Restore ordered touch control path

Author: DjDeveloperr

client/src/app/AppShell.tsx

@@ -39,10 +39,7 @@ import { useKeyboardInput } from "../features/input/useKeyboardInput";
 import { usePointerInput } from "../features/input/usePointerInput";
 import { simulatorRuntimeLabel } from "../features/simulators/simulatorDisplay";
 import { useSimulatorList } from "../features/simulators/useSimulatorList";
-import {
-  sendWebRtcControlMessage,
-  sendWebRtcRealtimeControlMessage,
-} from "../features/stream/streamWorkerClient";
+import { sendWebRtcControlMessage } from "../features/stream/streamWorkerClient";
 import { useLiveStream } from "../features/stream/useLiveStream";
 import { DebugPanel } from "../features/toolbar/DebugPanel";
 import { Toolbar } from "../features/toolbar/Toolbar";
@@ -846,11 +843,7 @@ export function AppShell({
         setAccessibilitySelectedId("");
         setAccessibilityHoveredId(null);
       }
-      sendControl(
-        selectedSimulator.udid,
-        { type: "touch", ...coords, phase },
-        { realtime: true },
-      );
+      sendControl(selectedSimulator.udid, { type: "touch", ...coords, phase });
     },
     onTouchPreview: showTouchIndicator,
     pan,
@@ -1002,20 +995,10 @@ export function AppShell({
     return state;
   }, []);
 
-  function sendControl(
-    udid: string,
-    message: ControlMessage,
-    options: { realtime?: boolean } = {},
-  ): boolean {
+  function sendControl(udid: string, message: ControlMessage): boolean {
     setLocalError("");
     const encoded = JSON.stringify(message);
-    const sent = options.realtime
-      ? sendWebRtcRealtimeControlMessage(encoded)
-      : sendWebRtcControlMessage(encoded);
-    if (sent) {
-      return true;
-    }
-    if (options.realtime) {
+    if (sendWebRtcControlMessage(encoded)) {
       return true;
     }
     const state = ensureControlSocket(udid);

client/src/features/input/usePointerInput.ts

@@ -33,7 +33,6 @@ export function usePointerInput({
   onTouchPreview,
 }: UsePointerInputOptions) {
   const activePointerRef = useRef<number | null>(null);
-  const sentMoveThisFrameRef = useRef(false);
   const moveFrameRef = useRef<number>(0);
   const panningRef = useRef<{
     startX: number;
@@ -53,34 +52,17 @@ export function usePointerInput({
   }, []);
 
   function queueMove(coords: Point) {
-    if (!sentMoveThisFrameRef.current) {
-      sentMoveThisFrameRef.current = true;
-      onTouch("moved", coords);
-      if (!moveFrameRef.current) {
-        moveFrameRef.current = requestAnimationFrame(() => {
-          moveFrameRef.current = 0;
-          sentMoveThisFrameRef.current = false;
-          const nextCoords = queuedMoveRef.current;
-          queuedMoveRef.current = null;
-          if (nextCoords) {
-            queueMove(nextCoords);
-          }
-        });
-      }
-      return;
-    }
     queuedMoveRef.current = coords;
     if (moveFrameRef.current) {
       return;
     }
 
     moveFrameRef.current = requestAnimationFrame(() => {
       moveFrameRef.current = 0;
-      sentMoveThisFrameRef.current = false;
       const nextCoords = queuedMoveRef.current;
       queuedMoveRef.current = null;
       if (nextCoords) {
-        queueMove(nextCoords);
+        onTouch("moved", nextCoords);
       }
     });
   }
@@ -91,7 +73,6 @@ export function usePointerInput({
       moveFrameRef.current = 0;
     }
     queuedMoveRef.current = null;
-    sentMoveThisFrameRef.current = false;
   }
 
   function startPanning(event: React.PointerEvent<HTMLElement>) {

client/src/features/stream/streamWorkerClient.ts

@@ -10,15 +10,12 @@ import type {
 
 const HAVE_CURRENT_DATA = 2;
 const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control";
-const WEBRTC_REALTIME_INPUT_CHANNEL_LABEL = "simdeck-input";
 const WEBRTC_TELEMETRY_CHANNEL_LABEL = "simdeck-telemetry";
 const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000;
 const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 8000;
 const WEBRTC_REMOTE_DISCONNECTED_GRACE_MS = 3000;
-const WEBRTC_REALTIME_INPUT_BUFFER_LIMIT = 1024;
 
 let activeWebRtcControlChannel: RTCDataChannel | null = null;
-let activeWebRtcRealtimeInputChannel: RTCDataChannel | null = null;
 let activeWebRtcTelemetryChannel: RTCDataChannel | null = null;
 let activeStreamClient: StreamWorkerClient | null = null;
 
@@ -28,20 +25,6 @@ export function sendWebRtcControlMessage(encoded: string): boolean {
   return sendDataChannelMessage(activeWebRtcControlChannel, encoded);
 }
 
-export function sendWebRtcRealtimeControlMessage(encoded: string): boolean {
-  if (activeWebRtcRealtimeInputChannel?.readyState !== "open") {
-    return false;
-  }
-  if (
-    activeWebRtcRealtimeInputChannel.bufferedAmount >
-    WEBRTC_REALTIME_INPUT_BUFFER_LIMIT
-  ) {
-    return true;
-  }
-  activeWebRtcRealtimeInputChannel.send(encoded);
-  return true;
-}
-
 export function sendWebRtcClientStats(stats: unknown): boolean {
   return sendDataChannelMessage(
     activeWebRtcTelemetryChannel,
@@ -90,7 +73,6 @@ class WebRtcStreamClient implements StreamClientBackend {
   private lastVideoFrameAt = 0;
   private peerConnection: RTCPeerConnection | null = null;
   private reconnectTimeout = 0;
-  private realtimeInputChannel: RTCDataChannel | null = null;
   private remoteMode = false;
   private reportedVideoConfig = false;
   private shouldReconnect = false;
@@ -159,20 +141,6 @@ class WebRtcStreamClient implements StreamClientBackend {
           activeWebRtcControlChannel = null;
         }
       });
-      const realtimeInputChannel = peerConnection.createDataChannel(
-        WEBRTC_REALTIME_INPUT_CHANNEL_LABEL,
-        {
-          maxRetransmits: 1,
-          ordered: false,
-        },
-      );
-      this.realtimeInputChannel = realtimeInputChannel;
-      activeWebRtcRealtimeInputChannel = realtimeInputChannel;
-      realtimeInputChannel.addEventListener("close", () => {
-        if (activeWebRtcRealtimeInputChannel === realtimeInputChannel) {
-          activeWebRtcRealtimeInputChannel = null;
-        }
-      });
       const telemetryChannel = peerConnection.createDataChannel(
         WEBRTC_TELEMETRY_CHANNEL_LABEL,
         {
@@ -345,11 +313,6 @@ class WebRtcStreamClient implements StreamClientBackend {
       activeWebRtcControlChannel = null;
     }
     this.controlChannel = null;
-    this.realtimeInputChannel?.close();
-    if (activeWebRtcRealtimeInputChannel === this.realtimeInputChannel) {
-      activeWebRtcRealtimeInputChannel = null;
-    }
-    this.realtimeInputChannel = null;
     this.telemetryChannel?.close();
     if (activeWebRtcTelemetryChannel === this.telemetryChannel) {
       activeWebRtcTelemetryChannel = null;

server/src/transport/webrtc.rs

@@ -36,7 +36,6 @@ use webrtc::track::track_local::TrackLocalWriter;
 const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1];
 const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302";
 const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control";
-const WEBRTC_REALTIME_INPUT_CHANNEL_LABEL: &str = "simdeck-input";
 const WEBRTC_TELEMETRY_CHANNEL_LABEL: &str = "simdeck-telemetry";
 const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(150);
 const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 3;
@@ -348,12 +347,7 @@ fn register_control_data_channel(
         let udid = udid.clone();
         Box::pin(async move {
             let label = channel.label();
-            if !matches!(
-                label.as_ref(),
-                WEBRTC_CONTROL_CHANNEL_LABEL
-                    | WEBRTC_REALTIME_INPUT_CHANNEL_LABEL
-                    | WEBRTC_TELEMETRY_CHANNEL_LABEL
-            ) {
+            if label != WEBRTC_CONTROL_CHANNEL_LABEL && label != WEBRTC_TELEMETRY_CHANNEL_LABEL {
                 return;
             }
             attach_control_data_channel(channel, session, state, udid);