diff --git a/native/runtime-host-peer/src/bindings.rs b/native/runtime-host-peer/src/bindings.rs index 9379a8b35d..d52de13197 100644 --- a/native/runtime-host-peer/src/bindings.rs +++ b/native/runtime-host-peer/src/bindings.rs @@ -18,6 +18,7 @@ */ use std::{ + collections::HashSet, path::PathBuf, sync::{Arc, Mutex, RwLock}, time::Duration, @@ -32,6 +33,7 @@ use crate::engine::{self, EngineCommand, PeerError, StreamCommand}; type IncomingStreamReceiver = mpsc::Receiver, PeerError>>; const IDENTITY_PAYLOAD_MAX_BYTES: usize = 8 * 1024; +const MAX_TRANSIT_PEERS: usize = 32; #[napi(object)] pub struct StartPeerEndpointOptions { @@ -48,9 +50,23 @@ pub struct ConnectPeerOptions { pub peer_id: String, pub route_hints: Vec, pub coordination_relays: Option>, + pub transit_relays: Option>, pub direct_deadline_ms: u32, } +#[napi(object)] +pub struct ConfigurePeerTransitOptions { + pub allowed_peer_ids: Vec, + pub trusted_relay_peer_ids: Vec, +} + +#[napi(object)] +pub struct PeerTransitSnapshot { + pub allowed_peer_count: u32, + pub active_reservation_count: u32, + pub active_circuit_count: u32, +} + #[napi(object)] pub struct PeerIdentitySignature { pub public_key: Buffer, @@ -62,6 +78,7 @@ pub struct PeerEndpoint { peer_id: String, listen_addresses: Vec, active_coordination_relays: Arc>>, + transit_snapshot: Arc>, commands: mpsc::Sender, incoming: Arc>>, mesh_incoming: Arc>>, @@ -89,6 +106,43 @@ impl PeerEndpoint { .unwrap_or_default() } + #[napi(getter)] + pub fn transit_snapshot(&self) -> PeerTransitSnapshot { + let snapshot = self + .transit_snapshot + .read() + .map(|snapshot| snapshot.clone()) + .unwrap_or_default(); + PeerTransitSnapshot { + allowed_peer_count: snapshot.allowed_peer_count as u32, + active_reservation_count: snapshot.active_reservation_count as u32, + active_circuit_count: snapshot.active_circuit_count as u32, + } + } + + #[napi] + pub async fn configure_transit(&self, options: ConfigurePeerTransitOptions) -> Result<()> { + let allowed_peers = parse_peer_ids(options.allowed_peer_ids)?; + let trusted_relays = parse_peer_ids(options.trusted_relay_peer_ids)?; + let local_peer_id = parse_peer_id(&self.peer_id)?; + if allowed_peers.contains(&local_peer_id) || trusted_relays.contains(&local_peer_id) { + return Err(Error::new( + Status::InvalidArg, + "peer endpoint cannot configure itself as a transit peer", + )); + } + let (result_tx, result_rx) = oneshot::channel(); + self.commands + .send(EngineCommand::ConfigureTransit { + allowed_peers, + trusted_relays, + result: result_tx, + }) + .await + .map_err(|_| native_closed_error())?; + result_rx.await.map_err(|_| native_closed_error()) + } + #[napi] pub async fn connect(&self, options: ConnectPeerOptions) -> Result { connect_peer(self, options, engine::StreamKind::Application).await @@ -173,6 +227,8 @@ async fn connect_peer( options.coordination_relays.unwrap_or_default(), "coordination relay", )?; + let transit_relays = + parse_addresses(options.transit_relays.unwrap_or_default(), "transit relay")?; if !(1..=120_000).contains(&options.direct_deadline_ms) { return Err(Error::new( Status::InvalidArg, @@ -188,6 +244,7 @@ async fn connect_peer( peer_id, route_hints, coordination_relays, + transit_relays, deadline: Duration::from_millis(u64::from(options.direct_deadline_ms)), }, stream_kind, @@ -304,6 +361,7 @@ pub fn start_peer_endpoint(options: StartPeerEndpointOptions) -> Result, label: &str) -> Result> { .collect() } +fn parse_peer_ids(values: Vec) -> Result> { + if values.len() > MAX_TRANSIT_PEERS { + return Err(Error::new( + Status::InvalidArg, + "transit policy cannot contain more than 32 peers", + )); + } + values + .into_iter() + .map(|value| parse_peer_id(&value)) + .collect() +} + fn validate_identity_payload(payload: &[u8]) -> Result<()> { if payload.is_empty() || payload.len() > IDENTITY_PAYLOAD_MAX_BYTES { return Err(Error::new( diff --git a/native/runtime-host-peer/src/engine.rs b/native/runtime-host-peer/src/engine.rs index 5adef1751b..7a433dba71 100644 --- a/native/runtime-host-peer/src/engine.rs +++ b/native/runtime-host-peer/src/engine.rs @@ -48,6 +48,7 @@ mod relay_discovery; use address::{ address_with_expected_peer, address_with_peer, coordination_relay_peer_id, is_relayed_address, + transit_relay_peer_id, }; use identity_store::load_or_create_key; use peer_stream::spawn_stream; @@ -71,6 +72,12 @@ const IDLE_CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); const TARGET_COORDINATION_RESERVATIONS: usize = 2; const MAX_AUTOMATIC_RELAY_CANDIDATES: usize = 8; const MAX_RELAY_ADDRESSES_PER_PEER: usize = 4; +const TRANSIT_FALLBACK_DELAY: Duration = Duration::from_secs(3); +const MAX_TRANSIT_RESERVATIONS: usize = 32; +const MAX_TRANSIT_CIRCUITS: usize = 8; +const MAX_TRANSIT_CIRCUITS_PER_PEER: usize = 2; +const MAX_TRANSIT_CIRCUIT_DURATION: Duration = Duration::from_secs(2 * 60 * 60); +const MAX_TRANSIT_CIRCUIT_BYTES: u64 = 256 * 1024 * 1024; #[derive(Clone)] pub struct StartOptions { @@ -85,6 +92,7 @@ pub struct StartedEndpoint { pub peer_id: PeerId, pub listen_addresses: Vec, pub active_coordination_relays: Arc>>, + pub transit_snapshot: Arc>, pub commands: mpsc::Sender, pub incoming: mpsc::Receiver, pub mesh_incoming: mpsc::Receiver, @@ -102,6 +110,7 @@ pub struct ConnectOptions { pub peer_id: PeerId, pub route_hints: Vec, pub coordination_relays: Vec, + pub transit_relays: Vec, pub deadline: Duration, } @@ -115,11 +124,23 @@ pub enum EngineCommand { request_id: u32, result: oneshot::Sender, }, + ConfigureTransit { + allowed_peers: HashSet, + trusted_relays: HashSet, + result: oneshot::Sender<()>, + }, Stop { result: oneshot::Sender<()>, }, } +#[derive(Clone, Default)] +pub struct TransitSnapshot { + pub allowed_peer_count: usize, + pub active_reservation_count: usize, + pub active_circuit_count: usize, +} + #[derive(Debug, Clone)] pub struct PeerError { pub code: &'static str, @@ -139,6 +160,7 @@ impl PeerError { struct Behaviour { connection_limits: connection_limits::Behaviour, relay_client: relay::client::Behaviour, + relay_server: relay::Behaviour, dcutr: dcutr::Behaviour, identify: identify::Behaviour, ping: ping::Behaviour, @@ -156,6 +178,9 @@ struct PendingConnect { direct_routes: Vec, coordination_relays: Vec, coordination_relay_peers: Vec, + transit_relays: Vec, + transit_relay_peers: HashSet, + transit_after: Instant, next_route_attempt: Instant, retry_coordination: bool, } @@ -168,13 +193,42 @@ pub enum StreamKind { #[derive(Clone, Copy, PartialEq, Eq)] pub(super) enum DialOrigin { - DirectRoute, - CoordinationRoute, + Direct, + Coordination, + Transit, } struct StartedConnect { direct_routes: Vec, coordination_relay_peers: Vec, + transit_relays: Vec, + transit_relay_peers: HashSet, +} + +struct TransitRuntime { + allowed_peers: Arc>>, + trusted_relays: Arc>>, + reservations: HashSet, + circuits: HashMap<(PeerId, PeerId), usize>, + listen_addresses: Vec, + published_addresses: Vec, + snapshot: Arc>, +} + +struct RouteRuntime<'a> { + active_coordination_relays: &'a Arc>>, + transit: &'a mut TransitRuntime, +} + +struct AllowedPeerLimiter(Arc>>); + +impl relay::RateLimiter for AllowedPeerLimiter { + fn try_next(&mut self, peer: PeerId, _: &Multiaddr, _: Instant) -> bool { + self.0 + .read() + .map(|allowed| allowed.contains(&peer)) + .unwrap_or(false) + } } #[derive(Default)] @@ -268,7 +322,10 @@ struct OpenedStream { } pub(super) enum StreamCompletion { - Application(ConnectionId), + Application { + connection_id: ConnectionId, + transit_relay_peer: Option, + }, MeshControl { coordination_relay_peers: Vec, }, @@ -325,6 +382,8 @@ pub fn start(options: StartOptions) -> Result { let (terminal_tx, terminal_rx) = mpsc::channel(1); let active_coordination_relays = Arc::new(RwLock::new(Vec::new())); let active_coordination_relays_for_thread = Arc::clone(&active_coordination_relays); + let transit_snapshot = Arc::new(RwLock::new(TransitSnapshot::default())); + let transit_snapshot_for_thread = Arc::clone(&transit_snapshot); let thread = thread::Builder::new() .name("maka-runtime-host-peer".to_owned()) .spawn(move || { @@ -335,6 +394,7 @@ pub fn start(options: StartOptions) -> Result { mesh_incoming_tx, ready_tx.clone(), active_coordination_relays_for_thread, + transit_snapshot_for_thread, ); if let Err(error) = result { let _ = ready_tx.send(Err(error.clone())); @@ -349,6 +409,7 @@ pub fn start(options: StartOptions) -> Result { peer_id: ready.0, listen_addresses: ready.1, active_coordination_relays, + transit_snapshot, commands: command_tx, incoming: incoming_rx, mesh_incoming: mesh_incoming_rx, @@ -364,6 +425,7 @@ fn run_endpoint( mesh_incoming_tx: mpsc::Sender, ready_tx: std::sync::mpsc::SyncSender), PeerError>>, active_coordination_relays: Arc>>, + transit_snapshot: Arc>, ) -> Result<(), PeerError> { let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -377,6 +439,7 @@ fn run_endpoint( mesh_incoming_tx, ready_tx, active_coordination_relays, + transit_snapshot, )) } @@ -387,6 +450,7 @@ async fn run_endpoint_async( mesh_incoming_tx: mpsc::Sender, ready_tx: std::sync::mpsc::SyncSender), PeerError>>, active_coordination_relays: Arc>>, + transit_snapshot: Arc>, ) -> Result<(), PeerError> { let key = match options.expected_peer_id { Some(expected) => { @@ -402,8 +466,23 @@ async fn run_endpoint_async( None => load_or_create_key(&options.key_path).await?, }; let local_peer_id = PeerId::from(key.public()); + let allowed_transit_peers = Arc::new(RwLock::new(HashSet::new())); + let trusted_transit_relays = Arc::new(RwLock::new(HashSet::new())); let (mut swarm, stream_control, mut incoming_streams, mesh_control, mut mesh_incoming) = - build_swarm(key)?; + build_swarm( + key, + Arc::clone(&allowed_transit_peers), + Arc::clone(&trusted_transit_relays), + )?; + let mut transit = TransitRuntime { + allowed_peers: allowed_transit_peers, + trusted_relays: trusted_transit_relays, + reservations: HashSet::new(), + circuits: HashMap::new(), + listen_addresses: Vec::new(), + published_addresses: Vec::new(), + snapshot: transit_snapshot, + }; let listen_addresses = if options.listen_addresses.is_empty() { vec![ @@ -456,6 +535,7 @@ async fn run_endpoint_async( event, &mut coordination_relays, &mut startup_external_candidate_ready, + &mut transit, ), Err(_) if pending_listeners.is_empty() => break, Err(_) => { @@ -468,13 +548,13 @@ async fn run_endpoint_async( } let mut bound_addresses = bound_addresses.into_iter().collect::>(); bound_addresses.sort_unstable_by_key(ToString::to_string); + transit.listen_addresses.clone_from(&bound_addresses); let _ = ready_tx.send(Ok((local_peer_id, bound_addresses))); let (opened_tx, mut opened_rx) = mpsc::channel::(COMMAND_CAPACITY); let (stream_completed_tx, mut stream_completed_rx) = mpsc::channel::(MAX_ESTABLISHED_CONNECTIONS as usize); let mut direct = DirectConnectState::default(); - let mut relayed = HashMap::>::new(); let mut external_candidate_ready = startup_external_candidate_ready; let mut deadline_tick = tokio::time::interval(Duration::from_millis(100)); deadline_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -511,10 +591,15 @@ async fn run_endpoint_async( } }; let request_id = options.request_id; + let transit_after = if started.direct_routes.is_empty() + && options.coordination_relays.is_empty() + { + Instant::now() + } else { + Instant::now() + TRANSIT_FALLBACK_DELAY + }; let retry_coordination = stream_kind == StreamKind::Application - && relayed - .get(&options.peer_id) - .is_some_and(|connections| !connections.is_empty()); + && stream_control.has_relayed_connection(options.peer_id); direct.pending.insert(request_id, PendingConnect { peer_id: options.peer_id, result, @@ -525,6 +610,9 @@ async fn run_endpoint_async( direct_routes: started.direct_routes, coordination_relays: options.coordination_relays, coordination_relay_peers: started.coordination_relay_peers, + transit_relays: started.transit_relays, + transit_relay_peers: started.transit_relay_peers, + transit_after, next_route_attempt: Instant::now(), retry_coordination, }); @@ -533,7 +621,6 @@ async fn run_endpoint_async( &mut direct, &coordination_relays, &stream_control, - &relayed, external_candidate_ready, Instant::now(), ); @@ -573,6 +660,20 @@ async fn run_endpoint_async( }; let _ = result.send(cancelled); } + Some(EngineCommand::ConfigureTransit { + allowed_peers, + trusted_relays, + result, + }) => { + configure_transit( + &mut swarm, + &stream_control, + &mut transit, + allowed_peers, + trusted_relays, + ); + let _ = result.send(()); + } Some(EngineCommand::Stop { result }) => { let _ = result.send(()); return Ok(()); @@ -621,8 +722,19 @@ async fn run_endpoint_async( } Some(completed) = stream_completed_rx.recv() => { match completed.kind { - StreamCompletion::Application(connection_id) => { + StreamCompletion::Application { + connection_id, + transit_relay_peer, + } => { direct.active.retain(|_, active| *active != connection_id); + if let Some(peer_id) = transit_relay_peer { + release_coordination_relays( + &mut swarm, + &mut coordination_relays, + &[peer_id], + &direct.active, + ); + } } StreamCompletion::MeshControl { coordination_relay_peers } => { release_coordination_relays( @@ -641,6 +753,7 @@ async fn run_endpoint_async( Ok(opened) => match waiter.stream_kind { StreamKind::Application => { let connection_id = opened.connection_id; + let transit_relay_peer = opened.relay_peer_id; retire_direct_dials( &mut swarm, &mut direct.retiring_connections, @@ -648,17 +761,26 @@ async fn run_endpoint_async( Some(connection_id), ); direct.active.insert(waiter.peer_id, connection_id); + let released_relays = waiter + .coordination_relay_peers + .iter() + .copied() + .filter(|peer_id| Some(*peer_id) != transit_relay_peer) + .collect::>(); release_coordination_relays( &mut swarm, &mut coordination_relays, - &waiter.coordination_relay_peers, + &released_relays, &direct.active, ); Ok(spawn_stream( waiter.peer_id, opened.stream, Some(( - StreamCompletion::Application(connection_id), + StreamCompletion::Application { + connection_id, + transit_relay_peer, + }, stream_completed_tx.clone(), )), )) @@ -697,6 +819,9 @@ async fn run_endpoint_async( &direct.active, ); let code = match waiter.stream_kind { + StreamKind::Application if !waiter.transit_relays.is_empty() => { + "transit_unavailable" + } StreamKind::Application => "direct_path_unavailable", StreamKind::MeshControl => "mesh_control_unavailable", }; @@ -710,11 +835,13 @@ async fn run_endpoint_async( handle_swarm_event( &mut swarm, event, - &mut relayed, &mut coordination_relays, &mut direct, &mut external_candidate_ready, - &active_coordination_relays, + RouteRuntime { + active_coordination_relays: &active_coordination_relays, + transit: &mut transit, + }, ); rebalance_automatic_relays( &mut swarm, @@ -761,7 +888,6 @@ async fn run_endpoint_async( &mut direct, &coordination_relays, &stream_control, - &relayed, external_candidate_ready, now, ); @@ -786,6 +912,10 @@ async fn run_endpoint_async( &direct.active, ); let (code, message) = match waiter.stream_kind { + StreamKind::Application if !waiter.transit_relays.is_empty() => ( + "transit_unavailable", + "no direct or approved transit path was established before the deadline", + ), StreamKind::Application => ( "direct_path_unavailable", "no direct path was established before the deadline", @@ -811,16 +941,20 @@ type BuiltSwarm = ( mpsc::Receiver, ); -fn build_swarm(key: identity::Keypair) -> Result { +fn build_swarm( + key: identity::Keypair, + allowed_transit_peers: Arc>>, + trusted_transit_relays: Arc>>, +) -> Result { let (application_stream, control, incoming) = application_stream::Behaviour::new( StreamProtocol::new(APPLICATION_PROTOCOL), INCOMING_STREAM_CAPACITY, - true, + Some(trusted_transit_relays), ); let (mesh_stream, mesh_control, mesh_incoming) = application_stream::Behaviour::new( StreamProtocol::new(MESH_CONTROL_PROTOCOL), MESH_INCOMING_STREAM_CAPACITY, - false, + None, ); let swarm = SwarmBuilder::with_existing_identity(key) .with_tokio() @@ -846,6 +980,10 @@ fn build_swarm(key: identity::Keypair) -> Result { .with_max_established_per_peer(Some(MAX_CONNECTIONS_PER_PEER)), ), relay_client, + relay_server: relay::Behaviour::new( + key.public().to_peer_id(), + transit_relay_config(allowed_transit_peers), + ), dcutr: dcutr::Behaviour::new(key.public().to_peer_id()), identify: identify::Behaviour::new(identify::Config::new( IDENTIFY_PROTOCOL.to_owned(), @@ -865,6 +1003,36 @@ fn build_swarm(key: identity::Keypair) -> Result { Ok((swarm, control, incoming, mesh_control, mesh_incoming)) } +fn transit_relay_config(allowed_peers: Arc>>) -> relay::Config { + let mut config = relay::Config { + max_reservations: MAX_TRANSIT_RESERVATIONS, + max_reservations_per_peer: relay_excess_limit(1), + reservation_duration: MAX_TRANSIT_CIRCUIT_DURATION, + max_circuits: MAX_TRANSIT_CIRCUITS, + max_circuits_per_peer: relay_excess_limit(MAX_TRANSIT_CIRCUITS_PER_PEER), + max_circuit_duration: MAX_TRANSIT_CIRCUIT_DURATION, + max_circuit_bytes: MAX_TRANSIT_CIRCUIT_BYTES, + ..relay::Config::default() + }; + let reservation_peers = Arc::clone(&allowed_peers); + config + .reservation_rate_limiters + .push(Box::new(AllowedPeerLimiter(reservation_peers))); + config + .circuit_src_rate_limiters + .push(Box::new(AllowedPeerLimiter(allowed_peers))); + config +} + +fn relay_excess_limit(maximum: usize) -> usize { + // libp2p-relay 0.21 rejects the next request only when the current count is + // greater than this value, so its configured boundary is one below the + // inclusive maximum exposed by Maka. + maximum + .checked_sub(1) + .expect("transit relay limits must be positive") +} + fn start_connect( swarm: &mut Swarm, coordination_relays: &mut HashMap, @@ -872,31 +1040,42 @@ fn start_connect( local_peer_id: PeerId, stream_kind: StreamKind, ) -> Result { - if options.route_hints.is_empty() && options.coordination_relays.is_empty() { + if options.route_hints.is_empty() + && options.coordination_relays.is_empty() + && options.transit_relays.is_empty() + { let code = match stream_kind { StreamKind::Application => "direct_path_unavailable", StreamKind::MeshControl => "mesh_control_unavailable", }; return Err(PeerError::new( code, - "the peer profile has no route hints or coordination relays", + "the peer profile has no direct, coordination, or transit route", )); } let mut relay_peers = Vec::new(); for relay_address in &options.coordination_relays { let relay_peer = coordination_relay_peer_id(relay_address)?; - if relay_peer == options.peer_id { - return Err(PeerError::new( - "coordination_unavailable", - "coordination relay cannot be the target peer", - )); - } - if relay_peer == local_peer_id { - return Err(PeerError::new( - "coordination_unavailable", - "peer endpoint cannot use itself as a coordination relay", - )); + validate_relay_target( + relay_peer, + options.peer_id, + local_peer_id, + "coordination_unavailable", + "coordination relay", + )?; + if !relay_peers.contains(&relay_peer) { + relay_peers.push(relay_peer); } + } + for relay_address in &options.transit_relays { + let relay_peer = transit_relay_peer_id(relay_address)?; + validate_relay_target( + relay_peer, + options.peer_id, + local_peer_id, + "transit_unavailable", + "transit relay", + )?; if !relay_peers.contains(&relay_peer) { relay_peers.push(relay_peer); } @@ -918,13 +1097,55 @@ fn start_connect( referenced.insert(relay_peer), )?; } + for relay_address in &options.transit_relays { + let relay_peer = transit_relay_peer_id(relay_address) + .expect("transit relay was validated before registration"); + register_coordination_relay( + coordination_relays, + relay_address, + local_peer_id, + false, + referenced.insert(relay_peer), + )?; + } maintain_coordination_relays(swarm, coordination_relays, false, Instant::now()); Ok(StartedConnect { direct_routes: direct_targets, coordination_relay_peers: relay_peers, + transit_relays: options.transit_relays.clone(), + transit_relay_peers: options + .transit_relays + .iter() + .map(|address| { + transit_relay_peer_id(address) + .expect("transit relay was validated before collection") + }) + .collect(), }) } +fn validate_relay_target( + relay_peer: PeerId, + target_peer: PeerId, + local_peer: PeerId, + error_code: &'static str, + label: &str, +) -> Result<(), PeerError> { + if relay_peer == target_peer { + return Err(PeerError::new( + error_code, + format!("{label} cannot be the target peer"), + )); + } + if relay_peer == local_peer { + return Err(PeerError::new( + error_code, + format!("peer endpoint cannot use itself as a {label}"), + )); + } + Ok(()) +} + fn maybe_open_peer_stream( request_id: u32, pending: &mut HashMap, @@ -937,11 +1158,17 @@ fn maybe_open_peer_stream( return; }; let peer_id = waiter.peer_id; + let eligible_relay_peers = match waiter.stream_kind { + StreamKind::Application => waiter.transit_relay_peers.clone(), + StreamKind::MeshControl => waiter.coordination_relay_peers.iter().copied().collect(), + }; let available = match waiter.stream_kind { StreamKind::Application => { - application_control.has_connection(peer_id, retiring_connections) + application_control.has_connection(peer_id, retiring_connections, &eligible_relay_peers) + } + StreamKind::MeshControl => { + mesh_control.has_connection(peer_id, retiring_connections, &eligible_relay_peers) } - StreamKind::MeshControl => mesh_control.has_connection(peer_id, retiring_connections), }; if waiter.opening.is_some() || !available { return; @@ -954,7 +1181,7 @@ fn maybe_open_peer_stream( StreamKind::MeshControl => &mut mesh_control, }; let result = control - .open_stream(peer_id, &retiring_connections) + .open_stream(peer_id, &retiring_connections, &eligible_relay_peers) .await .map_err(|error| error.to_string()); let _ = opened_tx.send(OpenedStream { request_id, result }).await; @@ -964,11 +1191,10 @@ fn maybe_open_peer_stream( fn handle_swarm_event( swarm: &mut Swarm, event: SwarmEvent, - relayed: &mut HashMap>, coordination_relays: &mut HashMap, direct: &mut DirectConnectState, external_candidate_ready: &mut bool, - active_coordination_relays: &Arc>>, + route_runtime: RouteRuntime<'_>, ) { match event { SwarmEvent::ConnectionEstablished { @@ -977,6 +1203,10 @@ fn handle_swarm_event( endpoint, .. } => { + discovery_debug(format_args!( + "connection established peer={peer_id} relayed={}", + endpoint.is_relayed() + )); if direct.retiring_connections.contains(&connection_id) { let _ = swarm.close_connection(connection_id); return; @@ -994,16 +1224,12 @@ fn handle_swarm_event( for connect in direct.pending.values_mut() { connect.dials.remove(&connection_id); } - if endpoint.is_relayed() { - relayed.entry(peer_id).or_default().insert(connection_id); - } } SwarmEvent::ConnectionClosed { peer_id, connection_id, .. } => { - remove_connection(relayed, peer_id, connection_id); direct.retiring_connections.remove(&connection_id); for connect in direct.pending.values_mut() { connect.dials.remove(&connection_id); @@ -1032,10 +1258,18 @@ fn handle_swarm_event( discard_automatic_relay_candidate(swarm, coordination_relays, peer_id); } if reservation_changed { - publish_active_coordination_relays(coordination_relays, active_coordination_relays); + publish_active_coordination_relays( + coordination_relays, + route_runtime.active_coordination_relays, + ); } } - SwarmEvent::OutgoingConnectionError { connection_id, .. } => { + SwarmEvent::OutgoingConnectionError { + connection_id, + error, + .. + } => { + discovery_debug(format_args!("outgoing connection failed: {error}")); direct.retiring_connections.remove(&connection_id); for connect in direct.pending.values_mut() { connect.dials.remove(&connection_id); @@ -1073,7 +1307,13 @@ fn handle_swarm_event( if let Some(relay) = coordination_relays.get_mut(&relay_peer_id) { relay.reservation_accepted = true; } - publish_active_coordination_relays(coordination_relays, active_coordination_relays); + publish_active_coordination_relays( + coordination_relays, + route_runtime.active_coordination_relays, + ); + } + SwarmEvent::Behaviour(BehaviourEvent::RelayServer(event)) => { + handle_transit_event(route_runtime.transit, event); } SwarmEvent::NewListenAddr { listener_id, @@ -1097,7 +1337,10 @@ fn handle_swarm_event( } discard_automatic_relay_candidate(swarm, coordination_relays, relay_peer); } - publish_active_coordination_relays(coordination_relays, active_coordination_relays); + publish_active_coordination_relays( + coordination_relays, + route_runtime.active_coordination_relays, + ); } } SwarmEvent::Behaviour(BehaviourEvent::Identify(identify::Event::Received { @@ -1165,7 +1408,10 @@ fn handle_swarm_event( discard_automatic_relay_candidate(swarm, coordination_relays, peer_id); } if changed { - publish_active_coordination_relays(coordination_relays, active_coordination_relays); + publish_active_coordination_relays( + coordination_relays, + route_runtime.active_coordination_relays, + ); } } _ => {} @@ -1177,18 +1423,135 @@ fn handle_startup_event( event: SwarmEvent, coordination_relays: &mut HashMap, external_candidate_ready: &mut bool, + transit: &mut TransitRuntime, ) { handle_swarm_event( swarm, event, - &mut HashMap::new(), coordination_relays, &mut DirectConnectState::default(), external_candidate_ready, - &Arc::new(RwLock::new(Vec::new())), + RouteRuntime { + active_coordination_relays: &Arc::new(RwLock::new(Vec::new())), + transit, + }, ); } +fn configure_transit( + swarm: &mut Swarm, + application_stream: &application_stream::Control, + transit: &mut TransitRuntime, + allowed_peers: HashSet, + trusted_relays: HashSet, +) { + let was_enabled = transit + .allowed_peers + .read() + .map(|current| !current.is_empty()) + .unwrap_or(false); + let enabled = !allowed_peers.is_empty(); + let removed = transit + .allowed_peers + .read() + .map(|current| { + current + .difference(&allowed_peers) + .copied() + .collect::>() + }) + .unwrap_or_default(); + let changed_relays = transit + .trusted_relays + .read() + .map(|current| { + current + .symmetric_difference(&trusted_relays) + .copied() + .collect::>() + }) + .unwrap_or_default(); + if let Ok(mut current) = transit.allowed_peers.write() { + *current = allowed_peers; + } + if let Ok(mut current) = transit.trusted_relays.write() { + *current = trusted_relays; + } + if enabled && !was_enabled { + let existing = swarm.external_addresses().cloned().collect::>(); + transit.published_addresses = transit + .listen_addresses + .iter() + .filter(|address| !existing.contains(*address)) + .cloned() + .collect(); + for address in &transit.published_addresses { + swarm.add_external_address(address.clone()); + } + } else if !enabled && was_enabled { + for address in transit.published_addresses.drain(..) { + swarm.remove_external_address(&address); + } + } + for peer_id in removed { + let _ = swarm.disconnect_peer_id(peer_id); + } + for connection_id in application_stream.connections_via(&changed_relays) { + let _ = swarm.close_connection(connection_id); + } + publish_transit_snapshot(transit); +} + +fn handle_transit_event(transit: &mut TransitRuntime, event: relay::Event) { + match event { + relay::Event::ReservationReqAccepted { src_peer_id, .. } => { + transit.reservations.insert(src_peer_id); + } + relay::Event::ReservationClosed { src_peer_id } + | relay::Event::ReservationTimedOut { src_peer_id } => { + transit.reservations.remove(&src_peer_id); + } + relay::Event::CircuitReqAccepted { + src_peer_id, + dst_peer_id, + } => { + *transit + .circuits + .entry((src_peer_id, dst_peer_id)) + .or_insert(0) += 1; + } + relay::Event::CircuitClosed { + src_peer_id, + dst_peer_id, + .. + } => { + if let Some(count) = transit.circuits.get_mut(&(src_peer_id, dst_peer_id)) { + *count -= 1; + if *count == 0 { + transit.circuits.remove(&(src_peer_id, dst_peer_id)); + } + } + } + _ => {} + } + publish_transit_snapshot(transit); +} + +fn publish_transit_snapshot(transit: &TransitRuntime) { + let allowed_peer_count = transit + .allowed_peers + .read() + .map(|peers| peers.len()) + .unwrap_or_default(); + if let Ok(mut snapshot) = transit.snapshot.write() { + *snapshot = TransitSnapshot { + allowed_peer_count, + active_reservation_count: transit.reservations.len(), + active_circuit_count: transit.circuits.values().sum(), + }; + } +} + fn register_coordination_relay( relays: &mut HashMap, address: &Multiaddr, @@ -1586,7 +1949,6 @@ fn retry_connect_routes( direct: &mut DirectConnectState, coordination_relays: &HashMap, stream_control: &application_stream::Control, - relayed: &HashMap>, external_candidate_ready: bool, now: Instant, ) { @@ -1595,51 +1957,76 @@ fn retry_connect_routes( if connect.next_route_attempt > now { continue; } - if stream_control.has_connection(peer_id, &direct.retiring_connections) { + if stream_control.has_connection( + peer_id, + &direct.retiring_connections, + &connect.transit_relay_peers, + ) { continue; } connect.next_route_attempt = now + COORDINATION_RETRY_INTERVAL; if !connect .dials .values() - .any(|origin| *origin == DialOrigin::DirectRoute) + .any(|origin| *origin == DialOrigin::Direct) && let Some(connection_id) = dial_direct_targets(swarm, peer_id, connect.direct_routes.clone()) { - connect.dials.insert(connection_id, DialOrigin::DirectRoute); + connect.dials.insert(connection_id, DialOrigin::Direct); } - if connect + if !connect .dials .values() - .any(|origin| *origin == DialOrigin::CoordinationRoute) - || (!connect.retry_coordination - && relayed.get(&peer_id).is_some_and(|ids| !ids.is_empty())) + .any(|origin| *origin == DialOrigin::Coordination) + && (connect.retry_coordination || !stream_control.has_relayed_connection(peer_id)) { - continue; - } - let mut targets = Vec::new(); - for relay in &connect.coordination_relays { - let relay_peer = coordination_relay_peer_id(relay) - .expect("coordination relay was validated before connecting"); - if !external_candidate_ready - || coordination_relays - .get(&relay_peer) - .is_none_or(|relay| !relay.identify_received || !relay.identify_sent) - { - continue; + let mut targets = Vec::new(); + for relay in &connect.coordination_relays { + let relay_peer = coordination_relay_peer_id(relay) + .expect("coordination relay was validated before connecting"); + if !external_candidate_ready + || coordination_relays + .get(&relay_peer) + .is_none_or(|relay| !relay.identify_received || !relay.identify_sent) + { + continue; + } + targets.push( + relay + .clone() + .with(Protocol::P2pCircuit) + .with(Protocol::P2p(peer_id)), + ); + } + if let Some(connection_id) = dial_direct_targets(swarm, peer_id, targets) { + connect + .dials + .insert(connection_id, DialOrigin::Coordination); + connect.retry_coordination = false; } - targets.push( - relay - .clone() - .with(Protocol::P2pCircuit) - .with(Protocol::P2p(peer_id)), - ); } - if let Some(connection_id) = dial_direct_targets(swarm, peer_id, targets) { - connect + + if now >= connect.transit_after + && !connect .dials - .insert(connection_id, DialOrigin::CoordinationRoute); - connect.retry_coordination = false; + .values() + .any(|origin| *origin == DialOrigin::Transit) + && let Some(connection_id) = dial_direct_targets( + swarm, + peer_id, + connect + .transit_relays + .iter() + .map(|relay| { + relay + .clone() + .with(Protocol::P2pCircuit) + .with(Protocol::P2p(peer_id)) + }) + .collect(), + ) + { + connect.dials.insert(connection_id, DialOrigin::Transit); } } } @@ -1675,20 +2062,6 @@ fn retire_direct_dials( } } -fn remove_connection( - connections: &mut HashMap>, - peer_id: PeerId, - connection_id: ConnectionId, -) { - let Some(ids) = connections.get_mut(&peer_id) else { - return; - }; - ids.remove(&connection_id); - if ids.is_empty() { - connections.remove(&peer_id); - } -} - fn native_error(error: impl std::fmt::Display) -> PeerError { PeerError::new("peer_native_failed", error.to_string()) } @@ -1789,6 +2162,108 @@ mod tests { std::fs::remove_dir_all(root).expect("remove test root"); } + #[tokio::test(flavor = "multi_thread")] + async fn approved_peers_can_exchange_an_application_stream_through_transit() { + let root = std::env::temp_dir().join(format!("maka-peer-transit-{}", PeerId::random())); + std::fs::create_dir_all(&root).expect("create test root"); + let relay = start(test_endpoint_options(root.join("relay.key"))).expect("start relay"); + let source = start(test_endpoint_options(root.join("source.key"))).expect("start source"); + let target_key = root.join("target.key"); + let target_peer_id = ensure_identity(target_key.clone()) + .await + .expect("create target identity"); + configure_test_transit( + &relay, + HashSet::from([source.peer_id, target_peer_id]), + HashSet::new(), + ) + .await; + let relay_address = relay + .listen_addresses + .first() + .expect("relay listen address") + .clone(); + let mut target_options = test_endpoint_options(target_key); + target_options.coordination_relays = vec![relay_address.clone()]; + let mut target = start(target_options).expect("start target"); + configure_test_transit(&target, HashSet::new(), HashSet::from([relay.peer_id])).await; + wait_for_test_snapshot(&relay, |snapshot| snapshot.active_reservation_count == 1).await; + + let (result, response) = oneshot::channel(); + source + .commands + .send(EngineCommand::Connect { + options: ConnectOptions { + request_id: 1, + peer_id: target.peer_id, + route_hints: Vec::new(), + coordination_relays: Vec::new(), + transit_relays: vec![relay_address], + deadline: Duration::from_secs(10), + }, + stream_kind: StreamKind::Application, + result, + }) + .await + .expect("send transit connect"); + let source_stream = tokio::time::timeout(Duration::from_secs(10), response) + .await + .expect("transit connect timeout") + .expect("transit connect response") + .expect("transit connect failed"); + let mut target_stream = + tokio::time::timeout(Duration::from_secs(5), target.incoming.recv()) + .await + .expect("transit inbound timeout") + .expect("transit inbound stream"); + + write_test_stream(&source_stream, b"through-transit").await; + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), target_stream.incoming.recv()) + .await + .expect("transit read timeout") + .expect("transit stream ended") + .expect("transit read failed"), + b"through-transit", + ); + assert_eq!( + relay + .transit_snapshot + .read() + .expect("read transit snapshot") + .active_circuit_count, + 1, + ); + + configure_test_transit(&relay, HashSet::from([target.peer_id]), HashSet::new()).await; + wait_for_test_snapshot(&relay, |snapshot| snapshot.active_circuit_count == 0).await; + let (result, response) = oneshot::channel(); + if source_stream + .commands + .send(StreamCommand::Write { + bytes: b"after-revocation".to_vec(), + result, + }) + .await + .is_ok() + { + let write = tokio::time::timeout(Duration::from_secs(2), response) + .await + .expect("revoked stream write timeout"); + assert!( + matches!(write, Err(_) | Ok(Err(_))), + "revoked transit stream remained writable", + ); + } + + close_test_stream(source_stream).await; + close_test_stream(target_stream).await; + stop_test_endpoint(source).await; + stop_test_endpoint(target).await; + stop_test_endpoint(relay).await; + std::fs::remove_dir_all(root).expect("remove test root"); + } + fn test_endpoint_options(key_path: PathBuf) -> StartOptions { StartOptions { key_path, @@ -1819,6 +2294,7 @@ mod tests { peer_id, route_hints: vec![route], coordination_relays: Vec::new(), + transit_relays: Vec::new(), deadline: Duration::from_secs(5), }, stream_kind, @@ -1875,6 +2351,45 @@ mod tests { endpoint.thread.join().expect("join endpoint thread"); } + async fn configure_test_transit( + endpoint: &StartedEndpoint, + allowed_peers: HashSet, + trusted_relays: HashSet, + ) { + let (result, response) = oneshot::channel(); + endpoint + .commands + .send(EngineCommand::ConfigureTransit { + allowed_peers, + trusted_relays, + result, + }) + .await + .expect("send transit policy"); + response.await.expect("apply transit policy"); + } + + async fn wait_for_test_snapshot( + endpoint: &StartedEndpoint, + ready: impl Fn(&TransitSnapshot) -> bool, + ) { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if endpoint + .transit_snapshot + .read() + .map(|snapshot| ready(&snapshot)) + .unwrap_or(false) + { + return; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("transit snapshot timeout"); + } + #[test] fn coordination_reservation_can_be_recreated_after_its_lifecycle_ends() { let now = Instant::now(); diff --git a/native/runtime-host-peer/src/engine/address.rs b/native/runtime-host-peer/src/engine/address.rs index 2feb7f64ab..008b9beda8 100644 --- a/native/runtime-host-peer/src/engine/address.rs +++ b/native/runtime-host-peer/src/engine/address.rs @@ -51,20 +51,32 @@ pub(super) fn peer_id_from_address(address: &Multiaddr) -> Option { } pub(super) fn coordination_relay_peer_id(address: &Multiaddr) -> Result { + base_relay_peer_id(address, "coordination_unavailable", "coordination relay") +} + +pub(super) fn transit_relay_peer_id(address: &Multiaddr) -> Result { + base_relay_peer_id(address, "transit_unavailable", "transit relay") +} + +fn base_relay_peer_id( + address: &Multiaddr, + error_code: &'static str, + label: &str, +) -> Result { let mut peer_id = None; for protocol in address.iter() { match protocol { Protocol::P2p(_) if peer_id.is_some() => { return Err(PeerError::new( - "coordination_unavailable", - "coordination relay address must name exactly one peer", + error_code, + format!("{label} address must name exactly one peer"), )); } Protocol::P2p(value) => peer_id = Some(value), Protocol::P2pCircuit => { return Err(PeerError::new( - "coordination_unavailable", - "coordination relay address must be a base relay address", + error_code, + format!("{label} address must be a base relay address"), )); } _ => {} @@ -73,8 +85,8 @@ pub(super) fn coordination_relay_peer_id(address: &Multiaddr) -> Result Ok(peer_id), _ => Err(PeerError::new( - "coordination_unavailable", - "coordination relay address must end with its peer identity", + error_code, + format!("{label} address must end with its peer identity"), )), } } @@ -84,3 +96,15 @@ pub(super) fn is_relayed_address(address: &Multiaddr) -> bool { .iter() .any(|protocol| matches!(protocol, Protocol::P2pCircuit)) } + +pub(super) fn relay_peer_id_from_circuit_address(address: &Multiaddr) -> Option { + let mut previous_peer = None; + for protocol in address.iter() { + match protocol { + Protocol::P2p(peer_id) => previous_peer = Some(peer_id), + Protocol::P2pCircuit => return previous_peer, + _ => {} + } + } + None +} diff --git a/native/runtime-host-peer/src/engine/application_stream.rs b/native/runtime-host-peer/src/engine/application_stream.rs index cc9eb0b7e6..5abd0e34dd 100644 --- a/native/runtime-host-peer/src/engine/application_stream.rs +++ b/native/runtime-host-peer/src/engine/application_stream.rs @@ -44,13 +44,13 @@ use libp2p::{ }; use tokio::sync::{mpsc, oneshot}; -use super::address::is_relayed_address; +use super::address::relay_peer_id_from_circuit_address; const OUTBOUND_COMMAND_CAPACITY: usize = 1; pub(super) struct Behaviour { protocol: StreamProtocol, - direct_only: bool, + trusted_transit_relays: Option>>>, incoming: mpsc::Sender, shared: Arc>, } @@ -64,14 +64,14 @@ impl Behaviour { pub(super) fn new( protocol: StreamProtocol, incoming_capacity: usize, - direct_only: bool, + trusted_transit_relays: Option>>>, ) -> (Self, Control, mpsc::Receiver) { let (incoming, receiver) = mpsc::channel(incoming_capacity); let shared = Arc::new(Mutex::new(DirectConnections::default())); ( Self { protocol, - direct_only, + trusted_transit_relays, incoming, shared: shared.clone(), }, @@ -80,12 +80,27 @@ impl Behaviour { ) } - fn handler(&mut self, connection_id: ConnectionId, peer_id: PeerId, relayed: bool) -> Handler { - if self.direct_only && relayed { + fn handler( + &mut self, + connection_id: ConnectionId, + peer_id: PeerId, + relay_peer_id: Option, + allow_relayed: bool, + ) -> Handler { + if relay_peer_id.is_some() + && !allow_relayed + && self.trusted_transit_relays.as_ref().is_some_and(|trusted| { + !trusted + .read() + .map(|peers| relay_peer_id.is_some_and(|peer| peers.contains(&peer))) + .unwrap_or(false) + }) + { + lock(&self.shared).insert(connection_id, peer_id, relay_peer_id, None); return Handler::relayed(); } let (sender, receiver) = mpsc::channel(OUTBOUND_COMMAND_CAPACITY); - lock(&self.shared).insert(connection_id, peer_id, sender); + lock(&self.shared).insert(connection_id, peer_id, relay_peer_id, Some(sender)); Handler::direct( peer_id, self.protocol.clone(), @@ -109,7 +124,9 @@ impl NetworkBehaviour for Behaviour { Ok(self.handler( connection_id, peer_id, - is_relayed_address(local_addr) || is_relayed_address(remote_addr), + relay_peer_id_from_circuit_address(local_addr) + .or_else(|| relay_peer_id_from_circuit_address(remote_addr)), + false, )) } @@ -121,7 +138,12 @@ impl NetworkBehaviour for Behaviour { _: Endpoint, _: PortUse, ) -> Result, ConnectionDenied> { - Ok(self.handler(connection_id, peer_id, is_relayed_address(address))) + Ok(self.handler( + connection_id, + peer_id, + relay_peer_id_from_circuit_address(address), + true, + )) } fn on_swarm_event(&mut self, event: FromSwarm) { @@ -150,18 +172,46 @@ pub(super) struct Control { } impl Control { - pub(super) fn has_connection(&self, peer_id: PeerId, excluded: &HashSet) -> bool { - lock(&self.shared).connection(peer_id, excluded).is_some() + pub(super) fn connections_via(&self, relays: &HashSet) -> Vec { + lock(&self.shared) + .connections + .iter() + .filter_map(|(connection_id, connection)| { + connection + .relay_peer_id + .is_some_and(|relay| relays.contains(&relay)) + .then_some(*connection_id) + }) + .collect() + } + + pub(super) fn has_connection( + &self, + peer_id: PeerId, + excluded: &HashSet, + allowed_relays: &HashSet, + ) -> bool { + lock(&self.shared) + .connection(peer_id, excluded, allowed_relays) + .is_some() + } + + pub(super) fn has_relayed_connection(&self, peer_id: PeerId) -> bool { + lock(&self.shared) + .connections + .values() + .any(|connection| connection.peer_id == peer_id && connection.relay_peer_id.is_some()) } pub(super) async fn open_stream( &mut self, peer_id: PeerId, excluded: &HashSet, + allowed_relays: &HashSet, ) -> Result { - let (connection_id, sender) = lock(&self.shared) - .connection(peer_id, excluded) - .ok_or(OpenStreamError::NoDirectConnection)?; + let (connection_id, relay_peer_id, sender) = lock(&self.shared) + .connection(peer_id, excluded, allowed_relays) + .ok_or(OpenStreamError::NoEligibleConnection)?; let (result, receiver) = oneshot::channel(); sender .send(NewStream { result }) @@ -172,6 +222,7 @@ impl Control { .map_err(|_| OpenStreamError::ConnectionClosed)??; Ok(OpenedStream { connection_id, + relay_peer_id, stream, }) } @@ -179,12 +230,13 @@ impl Control { pub(super) struct OpenedStream { pub(super) connection_id: ConnectionId, + pub(super) relay_peer_id: Option, pub(super) stream: Stream, } #[derive(Debug)] pub(super) enum OpenStreamError { - NoDirectConnection, + NoEligibleConnection, ConnectionClosed, UnsupportedProtocol, Io(io::Error), @@ -193,7 +245,12 @@ pub(super) enum OpenStreamError { impl std::fmt::Display for OpenStreamError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::NoDirectConnection => write!(formatter, "peer has no verified direct connection"), + Self::NoEligibleConnection => { + write!( + formatter, + "peer has no verified direct or approved transit connection" + ) + } Self::ConnectionClosed => write!(formatter, "direct connection closed"), Self::UnsupportedProtocol => { write!(formatter, "peer does not support the application protocol") @@ -210,7 +267,8 @@ struct DirectConnections { struct DirectConnection { peer_id: PeerId, - sender: mpsc::Sender, + relay_peer_id: Option, + sender: Option>, } impl DirectConnections { @@ -218,10 +276,17 @@ impl DirectConnections { &mut self, connection_id: ConnectionId, peer_id: PeerId, - sender: mpsc::Sender, + relay_peer_id: Option, + sender: Option>, ) { - self.connections - .insert(connection_id, DirectConnection { peer_id, sender }); + self.connections.insert( + connection_id, + DirectConnection { + peer_id, + relay_peer_id, + sender, + }, + ); } fn remove(&mut self, connection_id: ConnectionId) { @@ -232,15 +297,20 @@ impl DirectConnections { &self, peer_id: PeerId, excluded: &HashSet, - ) -> Option<(ConnectionId, mpsc::Sender)> { + allowed_relays: &HashSet, + ) -> Option<(ConnectionId, Option, mpsc::Sender)> { self.connections .iter() - .find(|(connection_id, connection)| { - connection.peer_id == peer_id + .find_map(|(connection_id, connection)| { + let sender = connection.sender.as_ref()?; + (connection.peer_id == peer_id && !excluded.contains(connection_id) - && !connection.sender.is_closed() + && connection + .relay_peer_id + .is_none_or(|relay| allowed_relays.contains(&relay)) + && !sender.is_closed()) + .then(|| (*connection_id, connection.relay_peer_id, sender.clone())) }) - .map(|(connection_id, connection)| (*connection_id, connection.sender.clone())) } } @@ -436,18 +506,24 @@ mod tests { use super::*; #[test] - fn application_protocol_is_registered_only_on_direct_connections() { + fn application_protocol_is_registered_only_on_direct_or_trusted_transit_connections() { let protocol = StreamProtocol::new("/maka/test/1"); let peer_id = PeerId::random(); - let (mut behaviour, control, _) = Behaviour::new(protocol.clone(), 1, true); + let relay_peer_id = PeerId::random(); + let trusted = Arc::new(std::sync::RwLock::new(HashSet::new())); + let (mut behaviour, control, _) = + Behaviour::new(protocol.clone(), 1, Some(Arc::clone(&trusted))); + let relay_address: Multiaddr = + format!("/ip4/127.0.0.1/udp/1/quic-v1/p2p/{relay_peer_id}/p2p-circuit") + .parse() + .expect("relay address"); let relayed = behaviour - .handle_established_outbound_connection( + .handle_established_inbound_connection( ConnectionId::new_unchecked(1), peer_id, - &"/memory/1/p2p-circuit".parse().expect("relay address"), - Endpoint::Dialer, - PortUse::Reuse, + &relay_address, + &relay_address, ) .expect("relayed handler"); assert_eq!( @@ -456,14 +532,41 @@ mod tests { ); assert!( lock(&control.shared) - .connection(peer_id, &HashSet::new()) + .connection(peer_id, &HashSet::new(), &HashSet::new()) .is_none() ); - assert!(!control.has_connection(peer_id, &HashSet::new())); + assert!(!control.has_connection(peer_id, &HashSet::new(), &HashSet::new())); + assert!(control.has_relayed_connection(peer_id)); + + trusted + .write() + .expect("trusted relays") + .insert(relay_peer_id); + let trusted_relay = behaviour + .handle_established_inbound_connection( + ConnectionId::new_unchecked(2), + peer_id, + &relay_address, + &relay_address, + ) + .expect("trusted relayed handler"); + assert_eq!( + trusted_relay + .listen_protocol() + .upgrade() + .protocol_info() + .collect::>(), + vec![protocol.clone()] + ); + assert!(control.has_connection(peer_id, &HashSet::new(), &HashSet::from([relay_peer_id]),)); + let relay_connections = control.connections_via(&HashSet::from([relay_peer_id])); + assert_eq!(relay_connections.len(), 2); + assert!(relay_connections.contains(&ConnectionId::new_unchecked(1))); + assert!(relay_connections.contains(&ConnectionId::new_unchecked(2))); let direct = behaviour .handle_established_outbound_connection( - ConnectionId::new_unchecked(2), + ConnectionId::new_unchecked(3), peer_id, &"/ip4/127.0.0.1/udp/1/quic-v1" .parse() @@ -482,12 +585,17 @@ mod tests { ); assert!( lock(&control.shared) - .connection(peer_id, &HashSet::new()) + .connection(peer_id, &HashSet::new(), &HashSet::new()) .is_some() ); - assert!(control.has_connection(peer_id, &HashSet::new())); - assert!( - !control.has_connection(peer_id, &HashSet::from([ConnectionId::new_unchecked(2)]),) - ); + assert!(control.has_connection(peer_id, &HashSet::new(), &HashSet::new())); + assert!(!control.has_connection( + peer_id, + &HashSet::from([ + ConnectionId::new_unchecked(2), + ConnectionId::new_unchecked(3), + ]), + &HashSet::new(), + )); } } diff --git a/native/runtime-host-peer/src/lib.rs b/native/runtime-host-peer/src/lib.rs index 8364692451..93e4348737 100644 --- a/native/runtime-host-peer/src/lib.rs +++ b/native/runtime-host-peer/src/lib.rs @@ -21,6 +21,7 @@ mod bindings; mod engine; pub use bindings::{ - ConnectPeerOptions, PeerEndpoint, PeerIdentitySignature, PeerStream, StartPeerEndpointOptions, - ensure_peer_identity, sign_peer_identity, start_peer_endpoint, verify_peer_identity, + ConfigurePeerTransitOptions, ConnectPeerOptions, PeerEndpoint, PeerIdentitySignature, + PeerStream, PeerTransitSnapshot, StartPeerEndpointOptions, ensure_peer_identity, + sign_peer_identity, start_peer_endpoint, verify_peer_identity, }; diff --git a/packages/runtime-host/src/__tests__/peer-listener.test.ts b/packages/runtime-host/src/__tests__/peer-listener.test.ts index d5eb0fc7dc..af7e9b7ab7 100644 --- a/packages/runtime-host/src/__tests__/peer-listener.test.ts +++ b/packages/runtime-host/src/__tests__/peer-listener.test.ts @@ -129,6 +129,7 @@ function peerWith(streams: RuntimeHostPeerNativeStream[]): RuntimeHostPeerClient throw new Error('not used'); }, verifyIdentity: () => false, + configureTransit: async () => undefined, connect: async () => { throw new Error('not used'); }, diff --git a/packages/runtime-host/src/__tests__/peer-native.test.ts b/packages/runtime-host/src/__tests__/peer-native.test.ts index 7dad8c1388..751e93f9c4 100644 --- a/packages/runtime-host/src/__tests__/peer-native.test.ts +++ b/packages/runtime-host/src/__tests__/peer-native.test.ts @@ -26,6 +26,7 @@ import { test } from 'node:test'; import { createRuntimeHostPeerClient } from '../client/peer-client.js'; import { ensureRuntimeHostPeerIdentity, + normalizePeerError, readRuntimeHostPeerAuthentication, readRuntimeHostPeerAuthenticationResult, RuntimeHostPeerError, @@ -33,6 +34,12 @@ import { type RuntimeHostPeerNativeStream, } from '../transport/peer-native.js'; +test('preserves transit route failures from the native boundary', () => { + const error = normalizePeerError(new Error('transit_unavailable: no approved route')); + assert.equal(error.code, 'transit_unavailable'); + assert.equal(error.message, 'no approved route'); +}); + test('shares one peer endpoint, serializes same-peer connects, and cancels independently', async () => { const directory = await mkdtemp(join(tmpdir(), 'maka-peer-abort-')); const nativePath = join(directory, 'peer.cjs'); @@ -61,16 +68,18 @@ module.exports = { peerId: 'client', listenAddresses: [], activeCoordinationRelays: [], - connect: ({ requestId, peerId, routeHints, coordinationRelays }) => { - stats.requests.push({ requestId, peerId, routeHints, coordinationRelays }); + transitSnapshot: { allowedPeerCount: 0, activeReservationCount: 0, activeCircuitCount: 0 }, + connect: ({ requestId, peerId, routeHints, coordinationRelays, transitRelays }) => { + stats.requests.push({ requestId, peerId, routeHints, coordinationRelays, transitRelays }); if (peerId === 'ready') return Promise.resolve(stream); return new Promise((resolve, reject) => pending.set(requestId, { resolve, reject })); }, - connectMeshControl: ({ requestId, peerId, routeHints, coordinationRelays }) => { - stats.requests.push({ requestId, peerId, routeHints, coordinationRelays }); + connectMeshControl: ({ requestId, peerId, routeHints, coordinationRelays, transitRelays }) => { + stats.requests.push({ requestId, peerId, routeHints, coordinationRelays, transitRelays }); if (peerId === 'ready') return Promise.resolve(stream); return new Promise((resolve, reject) => pending.set(requestId, { resolve, reject })); }, + configureTransit: async () => {}, cancelConnect: async (requestId) => { stats.cancellations.push(requestId); if (missFirstCancellation) { @@ -96,6 +105,7 @@ module.exports = { resolveRoutes: () => ({ routeHints: ['/memory/discovered'], coordinationRelays: ['/memory/relay'], + transitRelays: ['/memory/transit'], }), }, }); @@ -132,24 +142,28 @@ module.exports = { peerId: 'pending', routeHints: ['/memory/discovered', '/memory/1'], coordinationRelays: ['/memory/relay'], + transitRelays: ['/memory/transit'], }, { requestId: 2, peerId: 'shared', routeHints: ['/memory/discovered', '/memory/1'], coordinationRelays: ['/memory/relay'], + transitRelays: ['/memory/transit'], }, { requestId: 3, peerId: 'shared', routeHints: ['/memory/1'], coordinationRelays: [], + transitRelays: [], }, { requestId: 4, peerId: 'ready', routeHints: ['/memory/discovered', '/memory/1'], coordinationRelays: ['/memory/relay'], + transitRelays: ['/memory/transit'], }, ], cancellations: [1, 1], @@ -200,8 +214,10 @@ module.exports = { peerId: 'peer', listenAddresses: [], activeCoordinationRelays: [], + transitSnapshot: { allowedPeerCount: 0, activeReservationCount: 0, activeCircuitCount: 0 }, connect: async () => stream, connectMeshControl: async () => stream, + configureTransit: async () => {}, cancelConnect: async () => true, accept: async () => null, acceptMeshControl: async () => null, diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index 59020d7458..48df03de26 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -32,6 +32,7 @@ export interface RuntimeHostPeerConnectInput { readonly peerId: string; readonly routeHints: readonly string[]; readonly coordinationRelays?: readonly string[]; + readonly transitRelays?: readonly string[]; readonly directDeadlineMs: number; } @@ -40,6 +41,7 @@ export interface RuntimeHostPeerRouteResolver { | { readonly routeHints: readonly string[]; readonly coordinationRelays: readonly string[]; + readonly transitRelays?: readonly string[]; } | undefined; } @@ -52,6 +54,10 @@ export interface RuntimeHostPeerClient { }>; signIdentity(payload: Buffer): Promise; verifyIdentity(peerId: string, payload: Buffer, proof: RuntimeHostPeerIdentityProof): boolean; + configureTransit(input: { + readonly allowedPeerIds: readonly string[]; + readonly trustedRelayPeerIds: readonly string[]; + }): Promise; connect( input: RuntimeHostPeerConnectInput, signal?: AbortSignal, @@ -173,6 +179,13 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { }); } + configureTransit(input: { + readonly allowedPeerIds: readonly string[]; + readonly trustedRelayPeerIds: readonly string[]; + }): Promise { + return this.#requireEndpoint().configureTransit(input); + } + async connect( input: RuntimeHostPeerConnectInput, signal?: AbortSignal, @@ -287,6 +300,7 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { discovered?.coordinationRelays ?? [], input.coordinationRelays, ), + transitRelays: mergeAddresses(discovered?.transitRelays ?? [], input.transitRelays), requestId, }); let settled = false; diff --git a/packages/runtime-host/src/transport/peer-native.ts b/packages/runtime-host/src/transport/peer-native.ts index c9f362dc9a..449ee05331 100644 --- a/packages/runtime-host/src/transport/peer-native.ts +++ b/packages/runtime-host/src/transport/peer-native.ts @@ -31,6 +31,7 @@ export type RuntimeHostPeerErrorCode = | 'direct_path_unavailable' | 'mesh_control_unavailable' | 'coordination_unavailable' + | 'transit_unavailable' | 'peer_native_unavailable' | 'peer_native_failed' | 'peer_connect_in_progress'; @@ -63,11 +64,13 @@ export interface RuntimeHostPeerNativeEndpoint { readonly peerId: string; readonly listenAddresses: readonly string[]; readonly activeCoordinationRelays: readonly string[]; + readonly transitSnapshot: RuntimeHostPeerTransitSnapshot; connect(options: { readonly requestId: number; readonly peerId: string; readonly routeHints: readonly string[]; readonly coordinationRelays?: readonly string[]; + readonly transitRelays?: readonly string[]; readonly directDeadlineMs: number; }): Promise; connectMeshControl(options: { @@ -75,14 +78,25 @@ export interface RuntimeHostPeerNativeEndpoint { readonly peerId: string; readonly routeHints: readonly string[]; readonly coordinationRelays?: readonly string[]; + readonly transitRelays?: readonly string[]; readonly directDeadlineMs: number; }): Promise; + configureTransit(options: { + readonly allowedPeerIds: readonly string[]; + readonly trustedRelayPeerIds: readonly string[]; + }): Promise; cancelConnect(requestId: number): Promise; accept(): Promise; acceptMeshControl(): Promise; close(): Promise; } +export interface RuntimeHostPeerTransitSnapshot { + readonly allowedPeerCount: number; + readonly activeReservationCount: number; + readonly activeCircuitCount: number; +} + interface RuntimeHostPeerNativeModule { ensurePeerIdentity(keyPath: string): Promise; signPeerIdentity( @@ -400,7 +414,7 @@ export function normalizePeerError(error: unknown): RuntimeHostPeerError { if (error instanceof RuntimeHostPeerError) return error; const cause = asError(error); const match = - /^(peer_[a-z_]+|direct_path_unavailable|mesh_control_unavailable|coordination_unavailable):\s*(.*)$/su.exec( + /^(peer_[a-z_]+|direct_path_unavailable|mesh_control_unavailable|coordination_unavailable|transit_unavailable):\s*(.*)$/su.exec( cause.message, ); if (match && isPeerErrorCode(match[1])) { @@ -451,10 +465,14 @@ function isPeerNativeEndpoint(value: unknown): value is RuntimeHostPeerNativeEnd 'activeCoordinationRelays' in value && Array.isArray(value.activeCoordinationRelays) && value.activeCoordinationRelays.every((address) => typeof address === 'string') && + 'transitSnapshot' in value && + isPeerTransitSnapshot(value.transitSnapshot) && 'connect' in value && typeof value.connect === 'function' && 'connectMeshControl' in value && typeof value.connectMeshControl === 'function' && + 'configureTransit' in value && + typeof value.configureTransit === 'function' && 'cancelConnect' in value && typeof value.cancelConnect === 'function' && 'accept' in value && @@ -466,6 +484,23 @@ function isPeerNativeEndpoint(value: unknown): value is RuntimeHostPeerNativeEnd ); } +function isPeerTransitSnapshot(value: unknown): value is RuntimeHostPeerTransitSnapshot { + return ( + typeof value === 'object' && + value !== null && + 'allowedPeerCount' in value && + isCount(value.allowedPeerCount) && + 'activeReservationCount' in value && + isCount(value.activeReservationCount) && + 'activeCircuitCount' in value && + isCount(value.activeCircuitCount) + ); +} + +function isCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + function isAuthenticationPreface(value: unknown): value is { v: 1; credential: string } { return ( typeof value === 'object' && @@ -522,6 +557,7 @@ function isPeerErrorCode(value: string | undefined): value is RuntimeHostPeerErr value === 'direct_path_unavailable' || value === 'mesh_control_unavailable' || value === 'coordination_unavailable' || + value === 'transit_unavailable' || value === 'peer_native_unavailable' || value === 'peer_native_failed' || value === 'peer_connect_in_progress'