diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index 81e5fda205..5cec27e095 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -14,10 +14,6 @@ dependencies = [ "dstack[gateway] @ https://github.com/dstackai/dstack/archive/refs/heads/master.tar.gz", ] -[project.optional-dependencies] -# TODO: drop, unused by the server since 0.21.0 -sglang = ["sglang-router==0.3.2"] - [tool.setuptools.package-data] "dstack.gateway" = [ "resources/systemd/*", diff --git a/src/dstack/_internal/core/models/routers.py b/src/dstack/_internal/core/models/routers.py index 59545c658a..7d4acd169a 100644 --- a/src/dstack/_internal/core/models/routers.py +++ b/src/dstack/_internal/core/models/routers.py @@ -12,20 +12,6 @@ class RouterType(str, Enum): DYNAMO = "dynamo" -class SGLangServiceRouterConfig(CoreModel): # TODO: drop, unused by the server since 0.21.0 - type: Annotated[Literal["sglang"], Field(description="The router type")] = "sglang" - policy: Annotated[ - Literal["random", "round_robin", "cache_aware", "power_of_two"], - Field( - description="The routing policy. Options: `random`, `round_robin`, `cache_aware`, `power_of_two`" - ), - ] = "cache_aware" - pd_disaggregation: Annotated[ - bool, - Field(description="Enable PD disaggregation mode for the SGLang router"), - ] = False - - class ReplicaGroupRouterConfig(CoreModel): type: Annotated[ Literal["sglang", "dynamo"], @@ -38,6 +24,3 @@ class ReplicaGroupRouterConfig(CoreModel): ), ), ] = "sglang" - - -AnyServiceRouterConfig = SGLangServiceRouterConfig diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/router_workers.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/router_workers.jinja2 deleted file mode 100644 index 3af7ea612d..0000000000 --- a/src/dstack/_internal/proxy/gateway/resources/nginx/router_workers.jinja2 +++ /dev/null @@ -1,23 +0,0 @@ -{% for replica in replicas %} -# Worker {{ loop.index }} -upstream router_worker_{{ domain|replace('.', '_') }}_{{ ports[loop.index0] }}_upstream { - server unix:{{ replica.socket }}; -} - -server { - listen 127.0.0.1:{{ ports[loop.index0] }}; - access_log off; # disable access logs for this internal endpoint - - proxy_read_timeout 300s; - proxy_send_timeout 300s; - - location / { - proxy_pass http://router_worker_{{ domain|replace('.', '_') }}_{{ ports[loop.index0] }}_upstream; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Connection ""; - proxy_set_header Upgrade $http_upgrade; - } -} -{% endfor %} diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 index 5523dea849..7c8a5fcc92 100644 --- a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 +++ b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 @@ -4,13 +4,9 @@ limit_req_zone {{ zone.key }} zone={{ zone.name }}:10m rate={{ zone.rpm }}r/m; {% if replicas %} upstream {{ domain }}.upstream { - {% if router_port is not none %} - server 127.0.0.1:{{ router_port }}; # SGLang router on the gateway - {% else %} {% for replica in replicas %} server unix:{{ replica.socket }}; # replica {{ replica.id }} {% endfor %} - {% endif %} } {% else %} @@ -48,7 +44,7 @@ server { {% endfor %} {# For router services: block all requests except whitelisted locations added dynamically above #} - {% if has_router_replica or (router is not none and router.type == "sglang") %} + {% if has_router_replica %} location / { return 403; } diff --git a/src/dstack/_internal/proxy/gateway/routers/registry.py b/src/dstack/_internal/proxy/gateway/routers/registry.py index 5bf69675c8..c5fb7c0d6a 100644 --- a/src/dstack/_internal/proxy/gateway/routers/registry.py +++ b/src/dstack/_internal/proxy/gateway/routers/registry.py @@ -39,7 +39,6 @@ async def register_service( ssh_private_key=body.ssh_private_key, repo=repo, has_router_replica=body.has_router_replica, - router=body.router, nginx=nginx, service_conn_pool=service_conn_pool, ) diff --git a/src/dstack/_internal/proxy/gateway/schemas/registry.py b/src/dstack/_internal/proxy/gateway/schemas/registry.py index 89e60c65cf..9a800c2ceb 100644 --- a/src/dstack/_internal/proxy/gateway/schemas/registry.py +++ b/src/dstack/_internal/proxy/gateway/schemas/registry.py @@ -3,7 +3,6 @@ from pydantic import BaseModel, Field from dstack._internal.core.models.instances import SSHConnectionParams -from dstack._internal.core.models.routers import AnyServiceRouterConfig from dstack._internal.proxy.lib.models import RateLimit @@ -48,7 +47,6 @@ class RegisterServiceRequest(BaseModel): ssh_private_key: str rate_limits: tuple[RateLimit, ...] = () has_router_replica: bool = False - router: Optional[AnyServiceRouterConfig] = None class SetServiceIdRequest(BaseModel): diff --git a/src/dstack/_internal/proxy/gateway/services/model_routers/__init__.py b/src/dstack/_internal/proxy/gateway/services/model_routers/__init__.py deleted file mode 100644 index 43477d2d3f..0000000000 --- a/src/dstack/_internal/proxy/gateway/services/model_routers/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from dstack._internal.core.models.routers import AnyServiceRouterConfig, RouterType -from dstack._internal.proxy.gateway.services.model_routers.sglang import SglangRouter -from dstack._internal.proxy.lib.errors import ProxyError - -from .base import Router, RouterContext - - -def get_router(router: AnyServiceRouterConfig, context: RouterContext) -> Router: - if router.type == RouterType.SGLANG: - return SglangRouter(config=router, context=context) - raise ProxyError(f"Router type '{router.type}' is not available") - - -__all__ = [ - "Router", - "RouterContext", - "get_router", -] diff --git a/src/dstack/_internal/proxy/gateway/services/model_routers/base.py b/src/dstack/_internal/proxy/gateway/services/model_routers/base.py deleted file mode 100644 index a47677f400..0000000000 --- a/src/dstack/_internal/proxy/gateway/services/model_routers/base.py +++ /dev/null @@ -1,90 +0,0 @@ -from abc import ABC, abstractmethod -from pathlib import Path -from typing import List, Literal, Optional - -from pydantic import BaseModel, ConfigDict - -from dstack._internal.core.models.routers import AnyServiceRouterConfig - - -class RouterContext(BaseModel): - """Context for router initialization and configuration.""" - - model_config = ConfigDict(frozen=True) - - host: str = "127.0.0.1" - port: int - log_dir: Path - log_level: Literal["debug", "info", "warning", "error"] = "info" - - -class Router(ABC): - """Abstract base class for router implementations. - A router manages the lifecycle of worker replicas and handles request routing. - Different router implementations may have different mechanisms for managing - replicas. - """ - - def __init__( - self, - context: RouterContext, - config: Optional[AnyServiceRouterConfig] = None, - ): - """Initialize router with context. - - Args: - context: Runtime context for the router (host, port, logging, etc.) - config: Optional router configuration (implementation-specific) - """ - self.context = context - - @abstractmethod - def start(self) -> None: - """Start the router process. - - Raises: - Exception: If the router fails to start. - """ - ... - - @abstractmethod - def stop(self) -> None: - """Stop the router process. - - Raises: - Exception: If the router fails to stop. - """ - ... - - @abstractmethod - def is_running(self) -> bool: - """Check if the router is currently running and responding. - - Returns: - True if the router is running and healthy, False otherwise. - """ - ... - - @abstractmethod - def remove_replicas(self, replica_urls: List[str]) -> None: - """Unregister replicas from the router (actual API calls to remove workers). - - Args: - replica_urls: The list of replica URLs to remove from router. - - Raises: - Exception: If removing replicas fails. - """ - ... - - @abstractmethod - def update_replicas(self, replica_urls: List[str]) -> None: - """Update replicas for service, replacing the current set. - - Args: - replica_urls: The new list of replica URLs for this service. - - Raises: - Exception: If updating replicas fails. - """ - ... diff --git a/src/dstack/_internal/proxy/gateway/services/model_routers/sglang.py b/src/dstack/_internal/proxy/gateway/services/model_routers/sglang.py deleted file mode 100644 index c1c03c5a11..0000000000 --- a/src/dstack/_internal/proxy/gateway/services/model_routers/sglang.py +++ /dev/null @@ -1,325 +0,0 @@ -import shutil -import subprocess -import sys -import time -from typing import List, Optional - -import httpx -import psutil - -from dstack._internal.core.models.routers import AnyServiceRouterConfig, RouterType -from dstack._internal.proxy.lib.errors import UnexpectedProxyError -from dstack._internal.utils.logging import get_logger - -from .base import Router, RouterContext - -logger = get_logger(__name__) - - -class SglangRouter(Router): - """SGLang router implementation with 1:1 service-to-router.""" - - TYPE = RouterType.SGLANG - - def __init__(self, config: AnyServiceRouterConfig, context: RouterContext): - """Initialize SGLang router. - - Args: - config: SGLang router configuration (policy, cache_threshold, etc.) - context: Runtime context for the router (host, port, logging, etc.) - """ - super().__init__(context=context, config=config) - self.config = config - - def pid_from_tcp_ipv4_port(self, port: int) -> Optional[int]: - """ - Return PID of the process listening on the given TCP IPv4 port. - If no process is found, return None. - """ - for conn in psutil.net_connections(kind="tcp4"): - if conn.laddr and conn.laddr.port == port and conn.status == psutil.CONN_LISTEN: - return conn.pid - return None - - def start(self) -> None: - try: - logger.info("Starting sglang-router-new on port %s...", self.context.port) - - # Prometheus port is offset by 10000 from router port to keep it in a separate range - prometheus_port = self.context.port + 10000 - - cmd = [ - sys.executable, - "-m", - "sglang_router.launch_router", - "--host", - self.context.host, - "--port", - str(self.context.port), - "--prometheus-port", - str(prometheus_port), - "--prometheus-host", - self.context.host, - "--log-level", - self.context.log_level, - "--log-dir", - str(self.context.log_dir), - "--policy", - self.config.policy, - ] - if self.config.pd_disaggregation: - cmd.append("--pd-disaggregation") - - subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - time.sleep(2) - - if not self.is_running(): - raise UnexpectedProxyError( - f"Failed to start sglang router on port {self.context.port}" - ) - - logger.info( - "Sglang router started successfully on port %s (prometheus on %s)", - self.context.port, - prometheus_port, - ) - - except Exception: - logger.exception("Failed to start sglang-router") - raise - - def stop(self) -> None: - try: - pid = self.pid_from_tcp_ipv4_port(self.context.port) - - if pid: - logger.debug( - "Stopping sglang-router process (PID: %s) on port %s", - pid, - self.context.port, - ) - try: - proc = psutil.Process(pid) - proc.terminate() - try: - proc.wait(timeout=5) - except psutil.TimeoutExpired: - logger.warning( - "Process %s did not terminate gracefully, forcing kill", pid - ) - proc.kill() - except psutil.NoSuchProcess: - logger.debug("sglang-router process %s already exited before stop()", pid) - else: - logger.debug("No sglang-router process found on port %s", self.context.port) - - # Clean up router logs - if self.context.log_dir.exists(): - logger.debug("Cleaning up router logs for port %s...", self.context.port) - shutil.rmtree(self.context.log_dir, ignore_errors=True) - - except Exception: - logger.exception("Failed to stop sglang-router") - raise - - def is_running(self) -> bool: - """Check if the SGLang router is running and responding to HTTP requests on the assigned port.""" - try: - with httpx.Client(timeout=5.0) as client: - response = client.get(f"http://{self.context.host}:{self.context.port}/workers") - return response.status_code == 200 - except httpx.RequestError as e: - logger.debug( - "Sglang router not responding on port %s: %s", - self.context.port, - e, - ) - return False - - def remove_replicas(self, replica_urls: List[str]) -> None: - for replica_url in replica_urls: - self._remove_worker_from_router(replica_url) - - def update_replicas(self, replica_urls: List[str]) -> None: - """Update replicas for service, replacing the current set.""" - # Query router to get current worker URLs - current_workers = self._get_router_workers() - current_worker_urls: set[str] = set() - for worker in current_workers: - url = worker.get("url") - if url and isinstance(url, str): - # Normalize URL by removing trailing slashes to avoid path artifacts - normalized_url = url.rstrip("/") - current_worker_urls.add(normalized_url) - # Normalize target URLs to ensure consistent comparison - target_worker_urls = {url.rstrip("/") for url in replica_urls} - - # Workers to add - workers_to_add = target_worker_urls - current_worker_urls - # Workers to remove - workers_to_remove = current_worker_urls - target_worker_urls - - if workers_to_add: - logger.info( - "Sglang router update: adding %d workers for router on port %s", - len(workers_to_add), - self.context.port, - ) - if workers_to_remove: - logger.info( - "Sglang router update: removing %d workers for router on port %s", - len(workers_to_remove), - self.context.port, - ) - - # Add workers - for worker_url in sorted(workers_to_add): - success = self._register_worker(worker_url) - if not success: - logger.warning("Failed to add worker %s, continuing with others", worker_url) - - # Remove workers - for worker_url in sorted(workers_to_remove): - success = self._remove_worker_from_router(worker_url) - if not success: - logger.warning("Failed to remove worker %s, continuing with others", worker_url) - - def _get_router_workers(self) -> List[dict]: - try: - with httpx.Client(timeout=5.0) as client: - response = client.get(f"http://{self.context.host}:{self.context.port}/workers") - if response.status_code == 200: - response_data = response.json() - workers = response_data.get("workers", []) - return workers - return [] - except Exception: - logger.exception("Error getting sglang router workers") - return [] - - def _add_worker_to_router( - self, - url: str, - worker_type: str = "regular", - bootstrap_port: Optional[int] = None, - ) -> bool: - try: - payload: dict = {"url": url, "worker_type": worker_type} - if bootstrap_port is not None: - payload["bootstrap_port"] = bootstrap_port - with httpx.Client(timeout=5.0) as client: - response = client.post( - f"http://{self.context.host}:{self.context.port}/workers", - json=payload, - ) - if response.status_code == 202: - response_data = response.json() - if response_data.get("status") == "accepted": - logger.info( - "Worker %s (type=%s) accepted by sglang router on port %s", - url, - worker_type, - self.context.port, - ) - return True - else: - logger.error( - "Sglang router on port %s failed to accept worker: %s", - self.context.port, - response_data, - ) - return False - else: - logger.error( - "Failed to add worker %s: status %d, %s", - url, - response.status_code, - response.text, - ) - return False - except Exception: - logger.exception("Error adding worker %s", url) - return False - - def _register_worker(self, url: str) -> bool: - if not self.config.pd_disaggregation: - return self._add_worker_to_router(url, "regular", None) - - server_info_url = f"{url}/server_info" - try: - with httpx.Client(timeout=10) as client: - resp = client.get(server_info_url) - if resp.status_code != 200: - return False - data = resp.json() - if data.get("status") != "ready": - return False - disaggregation_mode = data.get("disaggregation_mode", "") - if disaggregation_mode == "prefill": - worker_type = "prefill" - bootstrap_port = data.get("disaggregation_bootstrap_port") - elif disaggregation_mode == "decode": - worker_type = "decode" - bootstrap_port = None - else: - worker_type = "regular" - bootstrap_port = None - logger.info( - "Registering worker %s (type=%s)", - url, - worker_type, - ) - return self._add_worker_to_router( - url, - worker_type, - bootstrap_port, - ) - except Exception: - logger.exception("Error registering worker %s", url) - return False - - def _remove_worker_from_router(self, worker_url: str) -> bool: - try: - current_workers = self._get_router_workers() - worker_id = None - for worker in current_workers: - url = worker.get("url") - if url and isinstance(url, str) and url == worker_url: - worker_id = worker.get("id") - if worker_id and isinstance(worker_id, str): - break - if not worker_id: - logger.error("No worker id found for url %s", worker_url) - return False - with httpx.Client(timeout=5.0) as client: - response = client.delete( - f"http://{self.context.host}:{self.context.port}/workers/{worker_id}" - ) - if response.status_code == 202: - response_data = response.json() - if response_data.get("status") == "accepted": - logger.info( - "Removed worker %s from sglang router on port %s", - worker_url, - self.context.port, - ) - return True - else: - logger.error( - "Sglang router on port %s failed to remove worker: %s", - self.context.port, - response_data, - ) - return False - else: - logger.error( - "Failed to remove worker %s: status %d, %s", - worker_url, - response.status_code, - response.text, - ) - return False - except Exception: - logger.exception("Error removing worker %s", worker_url) - return False diff --git a/src/dstack/_internal/proxy/gateway/services/nginx.py b/src/dstack/_internal/proxy/gateway/services/nginx.py index 60fcc49f77..29562c5556 100644 --- a/src/dstack/_internal/proxy/gateway/services/nginx.py +++ b/src/dstack/_internal/proxy/gateway/services/nginx.py @@ -1,24 +1,16 @@ import importlib.resources -import socket import subprocess import tempfile from asyncio import Lock from pathlib import Path -from typing import Dict, Optional -from urllib.parse import urlparse +from typing import Optional import jinja2 from pydantic import BaseModel from typing_extensions import Literal -from dstack._internal.core.models.routers import AnyServiceRouterConfig, RouterType from dstack._internal.proxy.gateway.const import PROXY_PORT_ON_GATEWAY from dstack._internal.proxy.gateway.models import ACMESettings -from dstack._internal.proxy.gateway.services.model_routers import ( - Router, - RouterContext, - get_router, -) from dstack._internal.proxy.lib import models from dstack._internal.proxy.lib.errors import ProxyError, UnexpectedProxyError from dstack._internal.utils.common import run_async @@ -75,8 +67,6 @@ class ServiceConfig(SiteConfig): locations: list[LocationConfig] replicas: list[ReplicaConfig] has_router_replica: bool = False - router: Optional[AnyServiceRouterConfig] = None - router_port: Optional[int] = None cors_enabled: bool = False @@ -91,18 +81,6 @@ class Nginx: def __init__(self, conf_dir: Path = Path("/etc/nginx/sites-enabled")) -> None: self._conf_dir = conf_dir self._lock: Lock = Lock() - # 1:1 service-to-router mapping - self._router_port_to_domain: Dict[int, str] = {} - self._domain_to_router: Dict[str, Router] = {} - self._ROUTER_PORT_MIN: int = 20000 - self._ROUTER_PORT_MAX: int = 24999 - self._WORKER_PORT_MIN: int = 10001 - self._WORKER_PORT_MAX: int = 11999 - self._next_router_port: int = self._ROUTER_PORT_MIN - # Tracking of worker ports to avoid conflicts across router instances - self._allocated_worker_ports: set[int] = set() - self._domain_to_worker_urls: Dict[str, list[str]] = {} - self._next_worker_port: int = self._WORKER_PORT_MIN async def register(self, conf: SiteConfig, acme: ACMESettings) -> None: logger.debug("Registering %s domain %s", conf.type, conf.domain) @@ -111,82 +89,6 @@ async def register(self, conf: SiteConfig, acme: ACMESettings) -> None: if conf.https: await run_async(self.run_certbot, conf.domain, acme) - if isinstance(conf, ServiceConfig) and conf.router and not conf.has_router_replica: - if conf.router.type == RouterType.SGLANG: - # Check if router already exists for this domain - if conf.domain in self._domain_to_router: - # Router already exists, reuse it - router = self._domain_to_router[conf.domain] - router_port = router.context.port - conf.router_port = router_port - else: - # Allocate router port for new router - router_port = self._allocate_router_port() - conf.router_port = router_port - - # Create per-service log directory - log_dir = Path(f"./router_logs/{conf.domain}") - - # Create router context with allocated port - ctx = RouterContext( - port=router_port, - log_dir=log_dir, - ) - - # Create new router instance for this service - router = get_router(conf.router, context=ctx) - - # Store mappings - self._router_port_to_domain[router_port] = conf.domain - self._domain_to_router[conf.domain] = router - - # Start router if not running - try: - if not await run_async(router.is_running): - await run_async(router.start) - except Exception: - # Clean up on failure - del self._router_port_to_domain[router_port] - del self._domain_to_router[conf.domain] - raise - - if conf.router.pd_disaggregation: - # PD path: replica_urls from internal_ip (router talks directly to workers) - if any(not r.internal_ip for r in conf.replicas): - raise ProxyError( - "PD disaggregation requires internal IP for all replicas." - ) - replica_urls = [ - f"http://{replica.internal_ip}:{replica.port}" - for replica in conf.replicas - ] - self._domain_to_worker_urls[conf.domain] = replica_urls - else: - # Non-PD path: allocate gateway-local ports, nginx proxies to replica sockets - allocated_ports = self._allocate_worker_ports(len(conf.replicas)) - replica_urls = [ - f"http://{router.context.host}:{port}" for port in allocated_ports - ] - if conf.replicas: - await run_async( - self.write_router_workers_conf, - conf, - allocated_ports, - ) - if conf.domain in self._domain_to_worker_urls: - self._discard_ports(self._domain_to_worker_urls[conf.domain]) - self._domain_to_worker_urls[conf.domain] = replica_urls - - try: - await run_async(router.update_replicas, replica_urls) - except Exception as e: - logger.exception( - "Failed to add replicas to router for domain=%s: %s", - conf.domain, - e, - ) - raise - await run_async(self.write_conf, conf.render(), conf_name) logger.info("Registered %s domain %s", conf.type, conf.domain) @@ -199,33 +101,6 @@ async def unregister(self, service: models.Service) -> None: return async with self._lock: await run_async(sudo_rm, conf_path) - - if domain in self._domain_to_router: - router = self._domain_to_router[domain] - # Remove all workers for this domain - if domain in self._domain_to_worker_urls: - worker_urls = self._domain_to_worker_urls[domain] - await run_async(router.remove_replicas, worker_urls) - pd_disaggregation = ( - service.router.pd_disaggregation if service.router else False - ) - if not pd_disaggregation: - self._discard_ports(worker_urls) - del self._domain_to_worker_urls[domain] - logger.debug("Removed worker URLs for domain %s", domain) - # Stop and kill the router - await run_async(router.stop) - # Remove from mappings - router_port = router.context.port - if router_port in self._router_port_to_domain: - del self._router_port_to_domain[router_port] - del self._domain_to_router[domain] - - # Remove workers config file - workers_conf_path = self._conf_dir / f"router-workers.{domain}.conf" - if workers_conf_path.exists(): - await run_async(sudo_rm, workers_conf_path) - await run_async(self.reload) logger.info("Unregistered domain %s", domain) @@ -300,153 +175,10 @@ def certificate_exists(domain: str) -> bool: def get_config_name(domain: str) -> str: return f"443-{domain}.conf" - @staticmethod - def _is_port_available(port: int) -> bool: - """Check if a port is actually available (not in use by any process). - - Tries to bind to the port to see if it's available. - """ - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - sock.bind(("127.0.0.1", port)) - # If bind succeeds, port is available - return True - except OSError: - # If bind fails (e.g., Address already in use), port is not available - return False - except Exception: - logger.warning("Error checking port %s availability", port) - return False - - def _allocate_router_port(self) -> int: - """Allocate next available router port in fixed range. - - Checks both our internal allocation map and actual port availability - to avoid conflicts with other services. Range chosen to avoid ephemeral ports. - """ - port = self._next_router_port - max_attempts = self._ROUTER_PORT_MAX - self._ROUTER_PORT_MIN + 1 - attempts = 0 - - while attempts < max_attempts: - # Check if port is already allocated by us - if port in self._router_port_to_domain: - port += 1 - if port > self._ROUTER_PORT_MAX: - port = self._ROUTER_PORT_MIN # Wrap around - attempts += 1 - continue - - # Check if port is actually available on the system - if self._is_port_available(port): - # Port is available, allocate it - self._next_router_port = port + 1 - if self._next_router_port > self._ROUTER_PORT_MAX: - self._next_router_port = self._ROUTER_PORT_MIN # Wrap around - logger.debug("Allocated router port %s", port) - return port - - # Port is in use, try next one - logger.debug("Port %s is in use, trying next port", port) - port += 1 - if port > self._ROUTER_PORT_MAX: - port = self._ROUTER_PORT_MIN # Wrap around - attempts += 1 - - raise UnexpectedProxyError( - f"Router port range exhausted ({self._ROUTER_PORT_MIN}-{self._ROUTER_PORT_MAX}). " - "All ports in range appear to be in use." - ) - - def _allocate_worker_ports(self, num_ports: int) -> list[int]: - """Allocate worker ports globally in fixed range. - - Worker ports are used by nginx to listen and proxy to worker sockets. - They must be unique across all router instances. Range chosen to avoid ephemeral ports. - - Args: - num_ports: Number of worker ports to allocate - - Returns: - List of allocated worker port numbers - """ - allocated = [] - port = self._next_worker_port - max_attempts = (self._WORKER_PORT_MAX - self._WORKER_PORT_MIN + 1) * 2 # Allow wrap-around - attempts = 0 - - while len(allocated) < num_ports and attempts < max_attempts: - # Check if port is already allocated globally - if port in self._allocated_worker_ports: - port += 1 - if port > self._WORKER_PORT_MAX: - port = self._WORKER_PORT_MIN # Wrap around - attempts += 1 - continue - - # Check if port is actually available on the system - if self._is_port_available(port): - allocated.append(port) - self._allocated_worker_ports.add(port) - logger.debug("Allocated worker port %s", port) - port += 1 - if port > self._WORKER_PORT_MAX: - port = self._WORKER_PORT_MIN # Wrap around - else: - logger.debug("Worker port %s is in use, trying next port", port) - port += 1 - if port > self._WORKER_PORT_MAX: - port = self._WORKER_PORT_MIN # Wrap around - - attempts += 1 - - if len(allocated) < num_ports: - # Free up the ports we did allocate - for p in allocated: - self._allocated_worker_ports.discard(p) - raise UnexpectedProxyError( - f"Failed to allocate {num_ports} worker ports in range " - f"({self._WORKER_PORT_MIN}-{self._WORKER_PORT_MAX}). " - f"Only allocated {len(allocated)} ports after {attempts} attempts." - ) - - # Update next worker port for next allocation - self._next_worker_port = port - if self._next_worker_port > self._WORKER_PORT_MAX: - self._next_worker_port = self._WORKER_PORT_MIN # Wrap around - - return allocated - - def _discard_ports(self, urls: list[str]) -> None: - for u in urls: - parsed = urlparse(u) - if parsed.port is not None and parsed.port in self._allocated_worker_ports: - self._allocated_worker_ports.discard(parsed.port) - def write_global_conf(self) -> None: conf = read_package_resource("00-log-format.conf") self.write_conf(conf, "00-log-format.conf") - def write_router_workers_conf(self, conf: ServiceConfig, allocated_ports: list[int]) -> None: - """Write router workers configuration file (generic).""" - # Pass ports to template - workers_config = generate_router_workers_config(conf, allocated_ports) - workers_conf_name = f"router-workers.{conf.domain}.conf" - self.write_conf(workers_config, workers_conf_name) - - -def generate_router_workers_config(conf: ServiceConfig, allocated_ports: list[int]) -> str: - """Generate router workers configuration (generic, uses router_workers.jinja2 template).""" - template = read_package_resource("router_workers.jinja2") - return jinja2.Template(template).render( - domain=conf.domain, - replicas=conf.replicas, - ports=allocated_ports, - proxy_port=PROXY_PORT_ON_GATEWAY, - ) - def read_package_resource(file: str) -> str: return ( diff --git a/src/dstack/_internal/proxy/gateway/services/registry.py b/src/dstack/_internal/proxy/gateway/services/registry.py index 3f296e6ca5..592a964924 100644 --- a/src/dstack/_internal/proxy/gateway/services/registry.py +++ b/src/dstack/_internal/proxy/gateway/services/registry.py @@ -6,7 +6,6 @@ import dstack._internal.proxy.gateway.schemas.registry as schemas from dstack._internal.core.models.instances import SSHConnectionParams -from dstack._internal.core.models.routers import AnyServiceRouterConfig, RouterType from dstack._internal.proxy.gateway import models as gateway_models from dstack._internal.proxy.gateway.const import SERVICE_ALREADY_REGISTERED_ERROR_TEMPLATE from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo @@ -49,7 +48,6 @@ async def register_service( nginx: Nginx, service_conn_pool: ServiceConnectionPool, has_router_replica: bool = False, - router: Optional[AnyServiceRouterConfig] = None, ) -> None: cors_enabled = model is not None and model.type == "chat" and model.format == "openai" service = models.Service( @@ -63,7 +61,6 @@ async def register_service( client_max_body_size=client_max_body_size, replicas=(), has_router_replica=has_router_replica, - router=router, cors_enabled=cors_enabled, ) @@ -268,11 +265,6 @@ async def register_model_entrypoint( logger.info("Entrypoint %s is now registered in project %s", domain, project_name) -def _uses_pd_disaggregation(service: models.Service) -> bool: - """PD disaggregation: router talks to replicas via internal_ip, no SSH tunnels needed.""" - return service.router is not None and service.router.pd_disaggregation - - async def apply_service( service: models.Service, old_service: Optional[models.Service], @@ -292,31 +284,18 @@ async def apply_service( ), service_conn_pool=service_conn_pool, ) - if _uses_pd_disaggregation(service): - replica_conns = {} - replica_failures = {} - replica_configs = [ - ReplicaConfig( - id=replica.id, - socket=Path("/dev/null"), - port=replica.app_port, - internal_ip=replica.internal_ip, - ) - for replica in service.replicas - ] - else: - replica_conns, replica_failures = await get_or_add_replica_connections( - service, repo, service_conn_pool + replica_conns, replica_failures = await get_or_add_replica_connections( + service, repo, service_conn_pool + ) + replica_configs = [ + ReplicaConfig( + id=replica.id, + socket=conn.app_socket_path, + port=replica.app_port, + internal_ip=replica.internal_ip, ) - replica_configs = [ - ReplicaConfig( - id=replica.id, - socket=conn.app_socket_path, - port=replica.app_port, - internal_ip=replica.internal_ip, - ) - for replica, conn in replica_conns.items() - ] + for replica, conn in replica_conns.items() + ] service_config = await get_nginx_service_config(service, replica_configs) await nginx.register(service_config, (await repo.get_config()).acme_settings) return replica_failures @@ -365,9 +344,6 @@ async def get_nginx_service_config( ) -> ServiceConfig: limit_req_zones: list[LimitReqZoneConfig] = [] locations: list[LocationConfig] = [] - is_router = ( - service.router is not None and service.router.type == RouterType.SGLANG - ) or service.has_router_replica sglang_limits: dict[str, LimitReqConfig] = {} sglang_prefix_lengths: dict[str, int] = {} # Track prefix lengths for most-specific selection @@ -382,7 +358,7 @@ async def get_nginx_service_config( limit_req_zones.append( LimitReqZoneConfig(name=zone_name, key=key, rpm=round(rate_limit.rps * 60)) ) - if is_router: + if service.has_router_replica: for path in ROUTER_WHITELISTED_PATHS: if rate_limit.prefix == path or path.startswith(rate_limit.prefix): # Use the longest prefix if multiple prefixes match the same path @@ -403,7 +379,7 @@ async def get_nginx_service_config( ) # Add router whitelisted paths as locations - if is_router: + if service.has_router_replica: for path in ROUTER_WHITELISTED_PATHS: # Use prefix match for paths that end with a slash and exact match for paths that don't if path.endswith("/"): @@ -414,7 +390,10 @@ async def get_nginx_service_config( ) # Don't auto-add / location for router-based services (catch-all 403 handles it) - if not any(location.prefix == "/" for location in locations) and not is_router: + if ( + not any(location.prefix == "/" for location in locations) + and not service.has_router_replica + ): locations.append(LocationConfig(prefix="/", limit_req=None)) return ServiceConfig( domain=service.domain_safe, @@ -427,7 +406,6 @@ async def get_nginx_service_config( locations=locations, replicas=sorted(replicas, key=lambda r: r.id), # sort for reproducible configs has_router_replica=service.has_router_replica, - router=service.router, cors_enabled=service.cors_enabled, ) diff --git a/src/dstack/_internal/proxy/lib/models.py b/src/dstack/_internal/proxy/lib/models.py index dbdc4d0381..53eb13e742 100644 --- a/src/dstack/_internal/proxy/lib/models.py +++ b/src/dstack/_internal/proxy/lib/models.py @@ -1,13 +1,12 @@ """Things stored in BaseProxyRepo implementations.""" from datetime import datetime -from typing import Iterable, Literal, Optional, Union +from typing import Any, Iterable, Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from typing_extensions import Annotated from dstack._internal.core.models.instances import SSHConnectionParams -from dstack._internal.core.models.routers import AnyServiceRouterConfig from dstack._internal.proxy.lib.errors import UnexpectedProxyError @@ -63,10 +62,16 @@ class Service(ImmutableModel): strip_prefix: bool = True # only used in-server replicas: tuple[Replica, ...] has_router_replica: bool = False - router: Optional[AnyServiceRouterConfig] = None - """TODO: drop `router`, unused by the server since 0.21.0""" cors_enabled: bool = False # only used on gateways; enabled for openai-format models + @model_validator(mode="before") + @classmethod + def _ignore_router(cls, data: Any) -> Any: + """Ignore the dropped `router` field for compatibility with 0.19.38-0.21.3 state files.""" + if isinstance(data, dict) and "router" in data: + data = {k: v for k, v in data.items() if k != "router"} + return data + @property def domain_safe(self) -> str: if self.domain is None: diff --git a/src/dstack/_internal/server/services/proxy/services/service_proxy.py b/src/dstack/_internal/server/services/proxy/services/service_proxy.py index ba74107b6b..0fa872be53 100644 --- a/src/dstack/_internal/server/services/proxy/services/service_proxy.py +++ b/src/dstack/_internal/server/services/proxy/services/service_proxy.py @@ -5,7 +5,6 @@ from fastapi import status from starlette.requests import ClientDisconnect -from dstack._internal.core.models.routers import RouterType from dstack._internal.proxy.lib.const import ROUTER_WHITELISTED_PATHS from dstack._internal.proxy.lib.deps import ProxyAuthContext from dstack._internal.proxy.lib.errors import ProxyError @@ -42,9 +41,7 @@ async def proxy( if not service.strip_prefix: path = concat_url_path(request.scope.get("root_path", "/"), request.url.path) - if ( - service.router is not None and service.router.type == RouterType.SGLANG - ) or service.has_router_replica: + if service.has_router_replica: path_for_match = path if path.startswith("/") else f"/{path}" if not _is_whitelisted_path(path_for_match, ROUTER_WHITELISTED_PATHS): raise ProxyError("Path is not allowed for this service", status.HTTP_403_FORBIDDEN)