Skip to content

Tune Nginx config on the gateway #4237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jvstme merged 1 commit into from
Sep 1, 2026
Merged

Tune Nginx config on the gateway #4237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions src/dstack/_internal/core/backends/base/compute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1098,12 +1098,6 @@ def get_gateway_user_data(authorized_key: str) -> str:
snap={"commands": [["install", "--classic", "certbot"]]},
runcmd=[
["ln", "-s", "/snap/bin/certbot", "/usr/bin/certbot"],
[
"sed",
"-i",
"s/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/",
"/etc/nginx/nginx.conf",
],
["su", "ubuntu", "-c", " && ".join(get_dstack_gateway_commands())],
],
ssh_authorized_keys=[authorized_key],
Expand Down
32 changes: 32 additions & 0 deletions src/dstack/_internal/proxy/gateway/resources/nginx/nginx.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

worker_rlimit_nofile 65535;

events {
worker_connections 16384;
}

http {
sendfile on;
tcp_nopush on;
types_hash_max_size 2048;

server_names_hash_bucket_size 128;

include /etc/nginx/mime.types;
default_type application/octet-stream;

ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;

access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;

gzip on;

include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
22 changes: 13 additions & 9 deletions src/dstack/_internal/proxy/gateway/services/nginx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

CERTBOT_TIMEOUT = 40
CERTBOT_2ND_TIMEOUT = 5
CONFIGS_DIR = Path("/etc/nginx/sites-enabled")
NGINX_DIR = Path("/etc/nginx")
logger = get_logger(__name__)


Expand Down Expand Up @@ -78,8 +78,10 @@ class ModelEntrypointConfig(SiteConfig):
class Nginx:
"""Updates nginx config and issues SSL certificates."""

def __init__(self, conf_dir: Path = Path("/etc/nginx/sites-enabled")) -> None:
self._conf_dir = conf_dir
def __init__(self, nginx_dir: Path = NGINX_DIR) -> None:
self._nginx_dir = nginx_dir
self._sites_enabled_dir = nginx_dir / "sites-enabled"
self._nginx_conf_path = nginx_dir / "nginx.conf"
self._lock: Lock = Lock()

async def register(self, conf: SiteConfig, acme: ACMESettings) -> None:
Expand All @@ -89,14 +91,14 @@ async def register(self, conf: SiteConfig, acme: ACMESettings) -> None:
if conf.https:
await run_async(self.run_certbot, conf.domain, acme)

await run_async(self.write_conf, conf.render(), conf_name)
await run_async(self.write_conf, conf.render(), self._sites_enabled_dir / conf_name)

logger.info("Registered %s domain %s", conf.type, conf.domain)

async def unregister(self, service: models.Service) -> None:
domain = service.domain_safe
logger.debug("Unregistering domain %s", domain)
conf_path = self._conf_dir / self.get_config_name(domain)
conf_path = self._sites_enabled_dir / self.get_config_name(domain)
if not conf_path.exists():
return
async with self._lock:
Expand All @@ -111,9 +113,8 @@ def reload() -> None:
if r.returncode != 0:
raise UnexpectedProxyError("Failed to reload nginx")

def write_conf(self, conf: str, conf_name: str) -> None:
def write_conf(self, conf: str, conf_path: Path) -> None:
"""Update config and reload nginx. Rollback changes on error."""
conf_path = self._conf_dir / conf_name
old_conf = conf_path.read_text() if conf_path.exists() else None
if conf == old_conf:
return
Expand Down Expand Up @@ -176,8 +177,11 @@ def get_config_name(domain: str) -> str:
return f"443-{domain}.conf"

def write_global_conf(self) -> None:
conf = read_package_resource("00-log-format.conf")
self.write_conf(conf, "00-log-format.conf")
log_format_conf = read_package_resource("00-log-format.conf")
self.write_conf(log_format_conf, self._sites_enabled_dir / "00-log-format.conf")

nginx_conf = read_package_resource("nginx.conf")
self.write_conf(nginx_conf, self._nginx_conf_path)


def read_package_resource(file: str) -> str:
Expand Down
53 changes: 26 additions & 27 deletions src/tests/_internal/proxy/gateway/routers/test_registry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@
from dstack._internal.proxy.lib.testing.common import make_project, make_service


def make_client(
nginx_conf_dir: Path, repo: Optional[GatewayProxyRepo] = None
) -> httpx.AsyncClient:
app = make_app(repo=repo or GatewayProxyRepo(), nginx=Nginx(conf_dir=nginx_conf_dir))
def make_client(nginx_dir: Path, repo: Optional[GatewayProxyRepo] = None) -> httpx.AsyncClient:
(nginx_dir / "sites-enabled").mkdir(exist_ok=True)
app = make_app(repo=repo or GatewayProxyRepo(), nginx=Nginx(nginx_dir=nginx_dir))
return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test/")


Expand Down Expand Up @@ -103,7 +102,7 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None:
)
assert resp.status_code == 200
assert resp.json() == {"status": "ok"}
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
# general
assert system_mocks.reload_nginx.call_count == 1
assert "server_name test-run.gtw.test;" in conf
Expand Down Expand Up @@ -136,7 +135,7 @@ async def test_register_with_https(self, tmp_path: Path, system_mocks: Mocks) ->
json=register_service_payload(domain="test-run.gtw.test", https=True),
)
assert resp.status_code == 200
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "listen 80;" in conf
assert "listen 443 ssl;" in conf
assert "ssl_certificate /etc/letsencrypt/live/test-run.gtw.test/fullchain.pem;" in conf
Expand All @@ -150,7 +149,7 @@ async def test_register_with_auth(self, tmp_path: Path, system_mocks: Mocks) ->
json=register_service_payload(domain="test-run.gtw.test", auth=True),
)
assert resp.status_code == 200
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "auth_request /_dstack_auth;" in conf
assert "proxy_pass http://localhost:8000/api/auth/test-proj;" in conf

Expand All @@ -167,8 +166,8 @@ async def test_register_same_name_error(self, tmp_path: Path, system_mocks: Mock
)
assert resp.status_code == 400
assert resp.json() == {"detail": "Service test-proj/test-run is already registered"}
assert (tmp_path / "443-test-run-1.gtw.test.conf").exists()
assert not (tmp_path / "443-test-run-2.gtw.test.conf").exists()
assert (tmp_path / "sites-enabled" / "443-test-run-1.gtw.test.conf").exists()
assert not (tmp_path / "sites-enabled" / "443-test-run-2.gtw.test.conf").exists()
assert system_mocks.reload_nginx.call_count == 1

async def test_register_same_name_in_different_projects(
Expand All @@ -185,8 +184,8 @@ async def test_register_same_name_in_different_projects(
json=register_service_payload(run_name="test-run", domain="test-run.proj-2.gtw.test"),
)
assert resp.status_code == 200
assert (tmp_path / "443-test-run.proj-1.gtw.test.conf").exists()
assert (tmp_path / "443-test-run.proj-2.gtw.test.conf").exists()
assert (tmp_path / "sites-enabled" / "443-test-run.proj-1.gtw.test.conf").exists()
assert (tmp_path / "sites-enabled" / "443-test-run.proj-2.gtw.test.conf").exists()

async def test_register_same_domain_error(self, tmp_path: Path, system_mocks: Mocks) -> None:
client = make_client(tmp_path)
Expand All @@ -203,7 +202,7 @@ async def test_register_same_domain_error(self, tmp_path: Path, system_mocks: Mo
assert resp.json() == {
"detail": "Domain name 'test-run.gtw.test' is already taken by another service"
}
assert (tmp_path / "443-test-run.gtw.test.conf").exists()
assert (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists()
assert system_mocks.reload_nginx.call_count == 1

@freeze_time(datetime(2024, 12, 12, 0, 30))
Expand Down Expand Up @@ -251,7 +250,7 @@ async def test_register_with_rate_limits(self, tmp_path: Path, system_mocks: Moc
),
)
assert resp.status_code == 200
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert (
"limit_req_zone $binary_remote_addr zone=0.test-run.gtw.test:10m rate=150r/m;" in conf
)
Expand All @@ -276,7 +275,7 @@ async def test_register_with_root_rate_limit(
),
)
assert resp.status_code == 200
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert (
"limit_req_zone $binary_remote_addr zone=0.test-run.gtw.test:10m rate=60r/m;" in conf
)
Expand All @@ -290,7 +289,7 @@ async def test_register_without_rate_limits(self, tmp_path: Path, system_mocks:
json=register_service_payload(domain="test-run.gtw.test", rate_limits=[]),
)
assert resp.status_code == 200
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "limit_req_zone" not in conf
assert "limit_req zone=" not in conf
assert "location / {" in conf
Expand All @@ -306,7 +305,7 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None:
json=register_service_payload(run_name="test-run", domain="test-run.gtw.test"),
)
assert resp.status_code == 200
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "upstream" not in conf
# register 2 replicas
resp = await client.post(
Expand All @@ -321,7 +320,7 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None:
)
assert resp.status_code == 200
assert resp.json() == {"status": "ok"}
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "upstream test-run.gtw.test.upstream" in conf
assert (m1 := re.search(r"server unix:/(.+)/replica.sock; # replica xxx-xxx", conf))
assert (m2 := re.search(r"server unix:/(.+)/replica.sock; # replica yyy-yyy", conf))
Expand Down Expand Up @@ -376,7 +375,7 @@ async def test_register_connection_error(self, tmp_path: Path, system_mocks: Moc
json=register_service_payload(run_name="test-run", domain="test-run.gtw.test"),
)
assert resp.status_code == 200
conf_before = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf_before = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
# register invalid replica
system_mocks.open_conn.side_effect = SSHError("test error")
resp = await client.post(
Expand All @@ -387,7 +386,7 @@ async def test_register_connection_error(self, tmp_path: Path, system_mocks: Moc
assert resp.json() == {
"detail": "Cannot register replica abc-def in service test-proj/test-run: test error"
}
conf_after = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf_after = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert conf_after == conf_before


Expand Down Expand Up @@ -450,12 +449,12 @@ async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None:
json=register_service_payload(run_name="test-run", domain="test-run.gtw.test"),
)
assert resp.status_code == 200
assert (tmp_path / "443-test-run.gtw.test.conf").exists()
assert (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists()
# unregister service
resp = await client.post("/api/registry/test-proj/services/test-run/unregister")
assert resp.status_code == 200
assert resp.json() == {"status": "ok"}
assert not (tmp_path / "443-test-run.gtw.test.conf").exists()
assert not (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists()
assert system_mocks.reload_nginx.call_count == 2

async def test_unregister_not_registered_error(
Expand Down Expand Up @@ -484,11 +483,11 @@ async def test_unregister_with_replicas(self, tmp_path: Path, system_mocks: Mock
json=register_replica_payload(job_id=job_id),
)
assert resp.status_code == 200
assert (tmp_path / "443-test-run.gtw.test.conf").exists()
assert (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists()
# unregister service
resp = await client.post("/api/registry/test-proj/services/test-run/unregister")
assert resp.status_code == 200
assert not (tmp_path / "443-test-run.gtw.test.conf").exists()
assert not (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists()
assert system_mocks.reload_nginx.call_count == 4
assert system_mocks.close_conn.call_count == 2

Expand Down Expand Up @@ -526,7 +525,7 @@ async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None:
json=register_replica_payload(job_id=job_id),
)
assert resp.status_code == 200
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "replica xxx-xxx" in conf
assert "replica yyy-yyy" in conf
# unregister 1 replica
Expand All @@ -535,7 +534,7 @@ async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None:
)
assert resp.status_code == 200
assert resp.json() == {"status": "ok"}
conf = (tmp_path / "443-test-run.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "replica xxx-xxx" in conf
assert "replica yyy-yyy" not in conf
assert system_mocks.reload_nginx.call_count == 4
Expand Down Expand Up @@ -585,7 +584,7 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None:
)
assert resp.status_code == 200
assert resp.json() == {"status": "ok"}
conf = (tmp_path / "443-gateway.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-gateway.gtw.test.conf").read_text()
assert "proxy_pass http://localhost:8000/api/models/test-proj/;" in conf
assert "listen 80;" in conf
assert "listen 443" not in conf
Expand All @@ -599,7 +598,7 @@ async def test_register_with_https(self, tmp_path: Path, system_mocks: Mocks) ->
json={"domain": "gateway.gtw.test", "https": True},
)
assert resp.status_code == 200
conf = (tmp_path / "443-gateway.gtw.test.conf").read_text()
conf = (tmp_path / "sites-enabled" / "443-gateway.gtw.test.conf").read_text()
assert "proxy_pass http://localhost:8000/api/models/test-proj/;" in conf
assert "listen 80;" in conf
assert "listen 443 ssl;" in conf
Expand Down
12 changes: 7 additions & 5 deletions src/tests/_internal/proxy/gateway/test_app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ async def test_lifespan(tmp_path: Path, system_mocks: Mocks) -> None:
make_service("test-proj", "test-run", domain="test-run.gtw.test", https=True)
)
nginx_dir = tmp_path / "nginx"
nginx_dir.mkdir()
app = make_app(repo=repo, nginx=Nginx(conf_dir=nginx_dir))
conf_dir = nginx_dir / "sites-enabled"
conf_dir.mkdir(parents=True)
app = make_app(repo=repo, nginx=Nginx(nginx_dir=nginx_dir))
async with lifespan(app):
assert (nginx_dir / "00-log-format.conf").exists()
assert (nginx_dir / "443-gateway.gtw.test.conf").exists()
assert (nginx_dir / "443-test-run.gtw.test.conf").exists()
assert (conf_dir / "00-log-format.conf").exists()
assert (conf_dir / "443-gateway.gtw.test.conf").exists()
assert (conf_dir / "443-test-run.gtw.test.conf").exists()
assert (nginx_dir / "nginx.conf").exists()
assert system_mocks.open_conn.call_count == 1
assert system_mocks.close_conn.call_count == 0
assert system_mocks.close_conn.call_count == 1
Loading