diff --git a/Cargo.lock b/Cargo.lock index c714293e7..4b2ffe18f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,7 +94,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -105,7 +105,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -386,6 +386,17 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -943,7 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1251,10 +1262,66 @@ dependencies = [ "tracing", ] +[[package]] +name = "harmonia-nar" +version = "3.0.0" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" +dependencies = [ + "bstr", + "bytes", + "derive_more", + "futures", + "harmonia-utils-io", + "nix", + "pin-project-lite", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "walkdir", +] + +[[package]] +name = "harmonia-protocol" +version = "3.0.0" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" +dependencies = [ + "async-stream", + "bstr", + "bytes", + "derive_more", + "futures", + "harmonia-nar", + "harmonia-protocol-derive", + "harmonia-store-core", + "harmonia-utils-hash", + "harmonia-utils-io", + "libc", + "num_enum", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "harmonia-protocol-derive" +version = "3.0.0" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "harmonia-store-aterm" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#2f09d761ed107b0f3079bc70a73dbfd240f846ee" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" dependencies = [ "bytes", "harmonia-store-core", @@ -1267,7 +1334,7 @@ dependencies = [ [[package]] name = "harmonia-store-core" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#2f09d761ed107b0f3079bc70a73dbfd240f846ee" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" dependencies = [ "bytes", "data-encoding", @@ -1285,10 +1352,26 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "harmonia-store-remote" +version = "3.0.0" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" +dependencies = [ + "async-stream", + "futures", + "harmonia-nar", + "harmonia-protocol", + "harmonia-store-core", + "harmonia-utils-io", + "prometheus", + "tokio", + "tracing", +] + [[package]] name = "harmonia-utils-base-encoding" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#2f09d761ed107b0f3079bc70a73dbfd240f846ee" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" dependencies = [ "data-encoding", "derive_more", @@ -1298,7 +1381,7 @@ dependencies = [ [[package]] name = "harmonia-utils-hash" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#2f09d761ed107b0f3079bc70a73dbfd240f846ee" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" dependencies = [ "data-encoding", "derive_more", @@ -1310,6 +1393,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "harmonia-utils-io" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#b9ec6540251dae513a509f119bca78e1562cd084" +dependencies = [ + "bytes", + "futures", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1491,6 +1586,43 @@ dependencies = [ "uuid", ] +[[package]] +name = "hydra-drv-daemon" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "db", + "fs-err", + "futures", + "harmonia-protocol", + "harmonia-store-core", + "harmonia-store-remote", + "harmonia-utils-hash", + "harmonia-utils-io", + "sqlx", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "hydra-evaluator" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "db", + "fs-err", + "futures", + "hydra-tracing", + "sqlx", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "hydra-queue-runner" version = "0.1.0" @@ -1853,7 +1985,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2182,7 +2314,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3176,7 +3308,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3547,7 +3679,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3887,7 +4019,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4701,7 +4833,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 57cbacf9a..e77f82a2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,12 @@ [workspace] -members = [ "subprojects/hydra-builder", "subprojects/hydra-queue-runner", "subprojects/crates/*" ] +members = [ + "subprojects/hydra-builder", + "subprojects/hydra-drv-daemon", + "subprojects/hydra-evaluator", + "subprojects/hydra-queue-runner", + "subprojects/crates/*", +] resolver = "2" [workspace.package] @@ -29,10 +35,13 @@ futures = "0.3" futures-util = "0.3" gethostname = "1" h2 = "0.4" +harmonia-protocol = { git = "https://github.com/nix-community/harmonia.git" } harmonia-store-aterm = { git = "https://github.com/nix-community/harmonia.git" } harmonia-store-core = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-remote = { git = "https://github.com/nix-community/harmonia.git" } harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia.git" } harmonia-utils-hash = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-io = { git = "https://github.com/nix-community/harmonia.git" } hashbrown = "0.16" http = "1.1" http-body-util = "0.1" @@ -94,7 +103,12 @@ uuid = "1.16" zerocopy = { version = "0.8", features = [ "derive" ] } [patch.crates-io] +harmonia-nar = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-protocol = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-protocol-derive = { git = "https://github.com/nix-community/harmonia.git" } harmonia-store-aterm = { git = "https://github.com/nix-community/harmonia.git" } harmonia-store-core = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-remote = { git = "https://github.com/nix-community/harmonia.git" } harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia.git" } harmonia-utils-hash = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-io = { git = "https://github.com/nix-community/harmonia.git" } diff --git a/Procfile b/Procfile index a970163c6..7a9d6379d 100644 --- a/Procfile +++ b/Procfile @@ -1,6 +1,7 @@ hydra-evaluator: ./foreman/start-evaluator.sh hydra-queue-runner: ./foreman/start-queue-runner.sh hydra-builder: ./foreman/start-builder.sh +hydra-drv-daemon: ./foreman/start-drv-daemon.sh hydra-notify: ./foreman/start-notify.sh hydra-server: ./foreman/start-hydra.sh manual: ./foreman/start-manual.sh diff --git a/flake.nix b/flake.nix index 51a0be014..40645449f 100644 --- a/flake.nix +++ b/flake.nix @@ -83,6 +83,10 @@ hydra-builder = self'.callPackage ./subprojects/hydra-builder/package.nix { inherit nixComponents; }; + hydra-drv-daemon = self'.callPackage ./subprojects/hydra-drv-daemon/package.nix { }; + hydra-evaluator = self'.callPackage ./subprojects/hydra-evaluator/package.nix { + inherit nixComponents; + }; }); mkHydraBuilder = { pkgs, nixComponents }: @@ -137,6 +141,8 @@ hydra-linters hydra-queue-runner hydra-builder + hydra-drv-daemon + hydra-evaluator ; }; @@ -153,6 +159,8 @@ builder = forEachSystemIncDarwin (system: packages.${system}.hydra-builder); + drvDaemon = forEachSystem (system: packages.${system}.hydra-drv-daemon); + nixosTests = import ./nixos-tests.nix { inherit forEachSystem nixpkgs nixosModules; }; @@ -280,6 +288,8 @@ hydra-linters hydra-queue-runner hydra-builder + hydra-evaluator + hydra-drv-daemon ; foreman = pkgs.callPackage ./packaging/foreman/package.nix { foreman-src = foreman; diff --git a/foreman/start-drv-daemon.sh b/foreman/start-drv-daemon.sh new file mode 100755 index 000000000..29991f713 --- /dev/null +++ b/foreman/start-drv-daemon.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. ./foreman/common.sh + +wait_for_postgres + +# wait until the hydra database exists (hydra-server creates it) +while ! psql -h "$HYDRA_PG_SOCKET_DIR" -p "$HYDRA_PG_PORT" -d hydra -c 'SELECT 1' >/dev/null 2>&1; do sleep 1; done + +wait_for_hydra_server + +DAEMON_SOCK="$HYDRA_DATA/drv-daemon.sock" +UPSTREAM_SOCK="${NIX_DAEMON_SOCKET_PATH:-/nix/var/nix/daemon-socket/socket}" + +export HYDRA_DBA="postgres://${USER}@localhost:$HYDRA_PG_PORT/hydra" + +exec hydra-drv-daemon \ + --socket "$DAEMON_SOCK" \ + --upstream-socket "$UPSTREAM_SOCK" diff --git a/foreman/start-evaluator.sh b/foreman/start-evaluator.sh index f813b5fc2..bbb7b8f02 100755 --- a/foreman/start-evaluator.sh +++ b/foreman/start-evaluator.sh @@ -9,4 +9,11 @@ wait_for_postgres wait_for_hydra_db wait_for_hydra_server +# Route IFD store ops through the drv-daemon when its socket is up. +# When the socket is missing the evaluator falls back to the local nix-daemon. +DAEMON_SOCK="$HYDRA_DATA/drv-daemon.sock" +if [ -S "$DAEMON_SOCK" ]; then + export NIX_REMOTE="unix://$DAEMON_SOCK" +fi + HYDRA_CONFIG=$HYDRA_DATA/hydra.conf exec hydra-evaluator diff --git a/foreman/start-hydra.sh b/foreman/start-hydra.sh index 3e9b92c5d..6d65ab531 100755 --- a/foreman/start-hydra.sh +++ b/foreman/start-hydra.sh @@ -24,10 +24,15 @@ hydra-create-user alice --password foobar --role admin if [ ! -f "$HYDRA_DATA/hydra.conf" ]; then echo "Creating a default hydra.conf" cat << EOF > "$HYDRA_DATA/hydra.conf" -# test-time instances likely don't want to bootstrap nixpkgs from scratch +# Use substitutes in dev instances to avoid bootstrapping nixpkgs. use-substitutes = true queue_runner_endpoint = http://localhost:$HYDRA_QUEUE_RUNNER_REST_PORT +# IFD evaluation is allowed in the dev shell so the Procfile drv-daemon +# (started by start-evaluator.sh via NIX_REMOTE) actually sees those +# builds. Drop this line to disable IFD entirely. +allow_import_from_derivation = true + listen_address = 127.0.0.1 diff --git a/nixos-modules/default.nix b/nixos-modules/default.nix index 38136d632..a07c9f2ff 100644 --- a/nixos-modules/default.nix +++ b/nixos-modules/default.nix @@ -7,6 +7,9 @@ rec { _file = ./default.nix; imports = [ ./web-app.nix ]; services.hydra-dev.package = lib.mkDefault flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra; + services.hydra-dev.evaluatorExecutable = lib.mkDefault "${ + flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-evaluator + }/bin/hydra-evaluator"; }; postgresql = ./postgresql.nix; @@ -31,6 +34,18 @@ rec { flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-builder; }; + drv-daemon = + { pkgs, lib, ... }: + { + _file = ./default.nix; + imports = [ ./drv-daemon-module.nix ]; + services.hydra-drv-daemon-dev.package = + lib.mkDefault + flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-drv-daemon; + }; + + evaluator-drv-daemon = ./evaluator-drv-daemon.nix; + hydra = { ... }: { @@ -39,6 +54,8 @@ rec { web-app queue-runner builder + drv-daemon + evaluator-drv-daemon ]; }; diff --git a/nixos-modules/drv-daemon-module.nix b/nixos-modules/drv-daemon-module.nix new file mode 100644 index 000000000..7185fa6bd --- /dev/null +++ b/nixos-modules/drv-daemon-module.nix @@ -0,0 +1,124 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.services.hydra-drv-daemon-dev; + user = "hydra-queue-runner"; +in +{ + options.services.hydra-drv-daemon-dev = { + enable = lib.mkEnableOption "Hydra drv-daemon (turns IFD / imperative builds into ad-hoc Hydra Builds)"; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.callPackage ../subprojects/hydra-drv-daemon/package.nix { }; + }; + + socketPath = lib.mkOption { + type = lib.types.path; + default = "/run/hydra-drv-daemon/socket"; + description = '' + Socket path used by clients (e.g. hydra-evaluator, imperative `nix-store --realise`) to reach the daemon. + ''; + }; + + upstreamSocket = lib.mkOption { + type = lib.types.path; + default = "/nix/var/nix/daemon-socket/socket"; + description = '' + Upstream nix-daemon socket that the drv-daemon proxies read + ops and `.drv` uploads to. + ''; + }; + + dbUrl = lib.mkOption { + type = lib.types.singleLineStr; + default = "postgres://hydra@%2Frun%2Fpostgresql:5432/hydra"; + description = "PostgreSQL connection URL."; + }; + }; + + config = lib.mkIf cfg.enable { + # The queue-runner module creates the shared user and PostgreSQL ident mapping. + assertions = [ + { + assertion = config.services.hydra-queue-runner-dev.enable; + message = '' + services.hydra-drv-daemon-dev.enable requires + services.hydra-queue-runner-dev.enable, because the daemon + shares the queue-runner's "hydra-queue-runner" user/group + and ident-map setup. + ''; + } + ]; + + systemd.services.hydra-drv-daemon = { + description = "Hydra drv-daemon (ad-hoc Build dispatcher)"; + + after = [ + "hydra-init.service" + "nix-daemon.socket" + "network.target" + ]; + requires = [ "hydra-init.service" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + RUST_BACKTRACE = "1"; + HYDRA_DBA = cfg.dbUrl; + }; + + serviceConfig = { + Restart = "always"; + RestartSec = "5s"; + + ExecStart = lib.escapeShellArgs [ + "${cfg.package}/bin/hydra-drv-daemon" + "--socket" + cfg.socketPath + "--upstream-socket" + cfg.upstreamSocket + ]; + + User = user; + Group = "hydra"; + + RuntimeDirectory = "hydra-drv-daemon"; + RuntimeDirectoryMode = "0755"; + + ReadWritePaths = [ + "/run/postgresql/.s.PGSQL.${toString config.services.postgresql.settings.port}" + cfg.upstreamSocket + ]; + ReadOnlyPaths = [ "/nix/" ]; + + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectClock = true; + ProtectHostname = true; + RestrictSUIDSGID = true; + RestrictRealtime = true; + RemoveIPC = true; + LockPersonality = true; + NoNewPrivileges = true; + UMask = "0022"; + CapabilityBoundingSet = ""; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + }; + }; + }; +} diff --git a/nixos-modules/evaluator-drv-daemon.nix b/nixos-modules/evaluator-drv-daemon.nix new file mode 100644 index 000000000..ead05abe2 --- /dev/null +++ b/nixos-modules/evaluator-drv-daemon.nix @@ -0,0 +1,72 @@ +{ + config, + lib, + ... +}: +let + cfg = config.services.hydra-evaluator-dev; + daemon = config.services.hydra-drv-daemon-dev; +in +{ + options.services.hydra-evaluator-dev = { + routeIfdsThroughDaemon = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Route hydra-evaluator's IFD store ops through hydra-drv-daemon + by setting `NIX_REMOTE` on the evaluator service. The daemon + creates an ad-hoc Build per IFD so the queue-runner / builder + pair handles the build instead of the evaluator host. + + Has no effect unless `services.hydra-drv-daemon-dev.enable` is + also set. + + Note: routing only takes effect if `allowImportFromDerivation` + is true; otherwise nix-eval-jobs rejects every IFD before the + daemon can see it. + ''; + }; + + allowImportFromDerivation = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Add `allow_import_from_derivation = true` to hydra.conf. + Untrusted Nix expressions can run builders during eval, which + is why this stays opt-in. + ''; + }; + }; + + config = lib.mkMerge [ + (lib.mkIf cfg.routeIfdsThroughDaemon { + assertions = [ + { + assertion = daemon.enable; + message = '' + services.hydra-evaluator-dev.routeIfdsThroughDaemon requires + services.hydra-drv-daemon-dev.enable. + ''; + } + ]; + + systemd.services.hydra-evaluator = { + after = [ "hydra-drv-daemon.service" ]; + requires = [ "hydra-drv-daemon.service" ]; + environment.NIX_REMOTE = "unix://${daemon.socketPath}"; + }; + + warnings = lib.optional (!cfg.allowImportFromDerivation) '' + services.hydra-evaluator-dev.routeIfdsThroughDaemon is enabled but + services.hydra-evaluator-dev.allowImportFromDerivation is not. + IFDs will be rejected by nix-eval-jobs before reaching the + daemon, so the routing has no effect. + ''; + }) + (lib.mkIf cfg.allowImportFromDerivation { + services.hydra-dev.extraConfig = '' + allow_import_from_derivation = true + ''; + }) + ]; +} diff --git a/nixos-modules/web-app.nix b/nixos-modules/web-app.nix index 4252a9765..5551db491 100644 --- a/nixos-modules/web-app.nix +++ b/nixos-modules/web-app.nix @@ -86,6 +86,11 @@ in description = "The Hydra package."; }; + evaluatorExecutable = mkOption { + type = types.path; + description = "Path to the hydra-evaluator executable."; + }; + hydraURL = mkOption { type = types.str; description = '' @@ -300,6 +305,9 @@ in ]; path = with pkgs; [ hostname-debian + # Because hydra-evaluator calls `hydra-eval-jobset`. If we + # move that perl script into rust, then we can get rid of + # this. cfg.package ]; environment = env // { @@ -307,11 +315,11 @@ in }; serviceConfig = { ExecStart = escapeShellArgs [ - "@${cfg.package}/bin/hydra-evaluator" + "@${cfg.evaluatorExecutable}" "hydra-evaluator" ]; ExecStopPost = escapeShellArgs [ - "${cfg.package}/bin/hydra-evaluator" + "${cfg.evaluatorExecutable}" "--unlock" ]; User = "hydra"; diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index efafd1973..fbd21019e 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -7,6 +7,8 @@ hydra-linters, hydra-queue-runner, hydra-builder, + hydra-drv-daemon, + hydra-evaluator, foreman, }: @@ -21,6 +23,8 @@ let hydra-linters hydra-queue-runner hydra-builder + hydra-drv-daemon + hydra-evaluator ]; # Collect and deduplicate build inputs from all components, diff --git a/subprojects/crates/db/src/connection.rs b/subprojects/crates/db/src/connection.rs index 6b3f1ea81..63c97df12 100644 --- a/subprojects/crates/db/src/connection.rs +++ b/subprojects/crates/db/src/connection.rs @@ -7,7 +7,7 @@ use harmonia_store_core::derived_path::OutputName; use harmonia_store_core::store_path::{StoreDir, StorePath}; use super::models::{ - Build, BuildSmall, BuildStatus, BuildSteps, InsertBuildMetric, InsertBuildProduct, + Build, BuildID, BuildSmall, BuildStatus, BuildSteps, InsertBuildMetric, InsertBuildProduct, InsertBuildStep, InsertBuildStepOutput, Jobset, UpdateBuild, UpdateBuildStep, UpdateBuildStepInFinish, }; @@ -429,6 +429,125 @@ impl Connection { }) .transpose() } + + /// Return finished ids so reconnect sweep can wake waiters for missed notifications. + #[tracing::instrument(skip(self, ids), err)] + pub async fn finished_build_ids(&mut self, ids: &[BuildID]) -> sqlx::Result> { + if ids.is_empty() { + return Ok(Vec::new()); + } + sqlx::query_scalar::<_, i32>("SELECT id FROM builds WHERE id = ANY($1) AND finished = 1") + .bind(ids) + .fetch_all(&mut *self.conn) + .await + } + + /// Ensure the hidden jobset used for daemon-submitted builds exists. + #[tracing::instrument(skip(self), err)] + pub async fn ensure_adhoc_jobset(&mut self) -> sqlx::Result { + let mut tx = self.conn.begin().await?; + + sqlx::query( + "INSERT INTO Users (userName, fullName, emailAddress, password) + VALUES ('adhoc', 'Ad-hoc', 'adhoc@localhost', '') + ON CONFLICT (userName) DO NOTHING", + ) + .execute(&mut *tx) + .await?; + + sqlx::query( + "INSERT INTO Projects (name, displayName, description, owner, enabled, hidden) + VALUES ('adhoc', 'Ad-hoc', 'Ad-hoc builds via drv-daemon', 'adhoc', 1, 1) + ON CONFLICT (name) DO NOTHING", + ) + .execute(&mut *tx) + .await?; + + sqlx::query( + "INSERT INTO Jobsets + (name, project, description, nixExprInput, nixExprPath, emailOverride, type, hidden) + VALUES ('adhoc', 'adhoc', 'Ad-hoc builds via drv-daemon', '', '', '', 0, 1) + ON CONFLICT (project, name) DO NOTHING", + ) + .execute(&mut *tx) + .await?; + + let id: i32 = + sqlx::query_scalar("SELECT id FROM Jobsets WHERE project = 'adhoc' AND name = 'adhoc'") + .fetch_one(&mut *tx) + .await?; + + tx.commit().await?; + Ok(id) + } + + /// Read a finished Build and top-level outputs, falling back to prior accepted outputs for cached duplicates. + #[tracing::instrument(skip(self), err)] + pub async fn get_finished_build( + &mut self, + build_id: BuildID, + ) -> sqlx::Result> { + let row = sqlx::query_as::<_, (Option, Option, Option, Option)>( + "SELECT buildStatus, startTime, stopTime, drvPath + FROM builds + WHERE id = $1 AND finished = 1", + ) + .bind(build_id) + .fetch_optional(&mut *self.conn) + .await?; + let Some((Some(status_int), start_time, stop_time, Some(drv_path))) = row else { + return Ok(None); + }; + let Some(status) = BuildStatus::from_i32(status_int) else { + return Ok(None); + }; + + let mut outputs = sqlx::query_as::<_, (String, Option)>( + "SELECT bso.name, bso.path + FROM buildstepoutputs bso + JOIN buildsteps bs ON bs.build = bso.build AND bs.stepnr = bso.stepnr + WHERE bso.build = $1 + AND bs.drvPath = $2 + AND bs.status = 0", + ) + .bind(build_id) + .bind(&drv_path) + .fetch_all(&mut *self.conn) + .await?; + + if outputs.is_empty() && status == BuildStatus::Success { + // Rejected builds can still have successful step rows; only reuse outputs from accepted builds. + outputs = sqlx::query_as::<_, (String, Option)>( + "SELECT DISTINCT ON (bso.name) bso.name, bso.path + FROM buildstepoutputs bso + JOIN buildsteps bs ON bs.build = bso.build AND bs.stepnr = bso.stepnr + JOIN builds b ON b.id = bs.build + WHERE bs.drvPath = $1 + AND bs.status = 0 + AND b.finished = 1 + AND b.buildStatus = 0 + ORDER BY bso.name, bs.build DESC, bs.stepnr DESC", + ) + .bind(&drv_path) + .fetch_all(&mut *self.conn) + .await?; + } + + Ok(Some(FinishedBuild { + status, + start_time, + stop_time, + outputs, + })) + } +} + +#[derive(Debug)] +pub struct FinishedBuild { + pub status: BuildStatus, + pub start_time: Option, + pub stop_time: Option, + pub outputs: Vec<(String, Option)>, } impl Transaction<'_> { @@ -437,6 +556,34 @@ impl Transaction<'_> { self.tx.commit().await } + /// Insert a kept ad-hoc Build; caller must register a waiter before commit. + #[tracing::instrument(skip(self), err)] + pub async fn insert_adhoc_build( + &mut self, + jobset_id: i32, + nix_name: &str, + drv_path: &str, + system: &str, + ) -> sqlx::Result { + let id: BuildID = sqlx::query_scalar( + "INSERT INTO Builds ( + finished, timestamp, jobset_id, job, nixname, drvPath, system, + maxsilent, timeout, ischannel, iscurrent, priority, globalpriority, keep + ) VALUES ( + 0, EXTRACT(EPOCH FROM NOW())::INT4, $1, $2, $3, $4, $5, + 7200, 36000, 0, 0, 100, 0, 1 + ) RETURNING id", + ) + .bind(jobset_id) + .bind(nix_name) + .bind(nix_name) + .bind(drv_path) + .bind(system) + .fetch_one(&mut *self.tx) + .await?; + Ok(id) + } + #[tracing::instrument(skip(self, v), err)] pub async fn update_build(&mut self, build_id: i32, v: UpdateBuild<'_>) -> sqlx::Result<()> { sqlx::query!( @@ -535,13 +682,15 @@ impl Transaction<'_> { path: &StorePath, ) -> sqlx::Result<()> { let path = store_dir.display(path).to_string(); - // TODO: support inserting multiple at the same time - sqlx::query!( - "UPDATE buildoutputs SET path = $3 WHERE build = $1 AND name = $2", - build_id, - name, - path.as_str(), + // Ad-hoc builds lack preinserted BuildOutputs rows, but hydra-update-gc-roots reads this table. + sqlx::query( + "INSERT INTO buildoutputs (build, name, path) + VALUES ($1, $2, $3) + ON CONFLICT (build, name) DO UPDATE SET path = EXCLUDED.path", ) + .bind(build_id) + .bind(name) + .bind(path.as_str()) .execute(&mut *self.tx) .await?; Ok(()) @@ -938,13 +1087,13 @@ impl Transaction<'_> { &mut self, store_dir: &StoreDir, start_time: Option, - build_id: crate::models::BuildID, + build_id: BuildID, drv_path: &StorePath, platform: Option<&str>, machine: String, status: BuildStatus, error_msg: Option, - propagated_from: Option, + propagated_from: Option, outputs: BTreeMap>, ) -> sqlx::Result { let step_nr = loop { @@ -1006,7 +1155,7 @@ impl Transaction<'_> { store_dir: &StoreDir, start_time: i32, stop_time: i32, - build_id: crate::models::BuildID, + build_id: BuildID, drv_path: &StorePath, outputs: BTreeMap, ) -> anyhow::Result { @@ -1060,7 +1209,7 @@ impl Transaction<'_> { store_dir: &StoreDir, start_time: i32, stop_time: i32, - build_id: crate::models::BuildID, + build_id: BuildID, drv_path: &StorePath, output: (OutputName, Option), ) -> anyhow::Result { @@ -1302,6 +1451,38 @@ mod tests { .unwrap(); } + async fn insert_build_row( + conn: &mut Connection, + id: i32, + drv_path: &StorePath, + finished: i32, + build_status: Option, + ) { + let sd = test_store_dir(); + // Finished Builds need start/stop times to satisfy schema checks. + let (start_time, stop_time) = if finished == 1 { + (Some(1), Some(2)) + } else { + (None, None) + }; + sqlx::query( + "INSERT INTO Builds ( + id, finished, timestamp, jobset_id, job, nixname, drvPath, system, + maxsilent, timeout, ischannel, iscurrent, priority, globalpriority, keep, + startTime, stopTime, buildStatus + ) VALUES ($1, $2, 0, 1, 'job', 'nix', $3, '', 0, 0, 0, 0, 0, 0, 0, $4, $5, $6)", + ) + .bind(id) + .bind(finished) + .bind(sd.display(drv_path).to_string()) + .bind(start_time) + .bind(stop_time) + .bind(build_status) + .execute(&mut *conn.conn) + .await + .unwrap(); + } + #[tokio::test] async fn resolve_depth_1() { let (_pg, mut conn) = setup().await; @@ -1597,4 +1778,132 @@ mod tests { .await; assert_eq!(result, Some(sp("final-result"))); } + + fn store_str(path: &StorePath) -> String { + test_store_dir().display(path).to_string() + } + + #[tokio::test] + async fn get_finished_build_returns_none_while_unfinished() { + let (_pg, mut conn) = setup().await; + insert_build_row(&mut conn, 1, &sp("foo.drv"), 0, None).await; + assert!(conn.get_finished_build(1).await.unwrap().is_none()); + } + + #[tokio::test] + async fn get_finished_build_uses_own_step_outputs() { + let (_pg, mut conn) = setup().await; + insert_build_row(&mut conn, 1, &sp("foo.drv"), 1, Some(0)).await; + insert_step(&mut conn, 1, 1, &sp("foo.drv")).await; + insert_output(&mut conn, 1, 1, "out", &sp("result")).await; + + let r = conn.get_finished_build(1).await.unwrap().unwrap(); + assert_eq!(r.status, BuildStatus::Success); + assert_eq!( + r.outputs, + vec![("out".to_owned(), Some(store_str(&sp("result"))))], + ); + } + + #[tokio::test] + async fn get_finished_build_falls_back_when_own_steps_missing() { + let (_pg, mut conn) = setup().await; + insert_build_row(&mut conn, 1, &sp("foo.drv"), 1, Some(0)).await; + insert_step(&mut conn, 1, 1, &sp("foo.drv")).await; + insert_output(&mut conn, 1, 1, "out", &sp("result")).await; + insert_build_row(&mut conn, 2, &sp("foo.drv"), 1, Some(0)).await; + + let r = conn.get_finished_build(2).await.unwrap().unwrap(); + assert_eq!( + r.outputs, + vec![("out".to_owned(), Some(store_str(&sp("result"))))], + ); + } + + #[tokio::test] + async fn get_finished_build_skips_rejected_source_in_fallback() { + // Rejected source builds can still have successful step outputs. + let (_pg, mut conn) = setup().await; + insert_build_row(&mut conn, 1, &sp("foo.drv"), 1, Some(11)).await; + insert_step(&mut conn, 1, 1, &sp("foo.drv")).await; + insert_output(&mut conn, 1, 1, "out", &sp("result-rejected")).await; + insert_build_row(&mut conn, 2, &sp("foo.drv"), 1, Some(0)).await; + + let r = conn.get_finished_build(2).await.unwrap().unwrap(); + assert_eq!(r.status, BuildStatus::Success); + assert!( + r.outputs.is_empty(), + "fallback should not surface outputs from rejected Build #1, got {:?}", + r.outputs + ); + } + + #[tokio::test] + async fn get_finished_build_fallback_picks_most_recent_successful() { + let (_pg, mut conn) = setup().await; + insert_build_row(&mut conn, 1, &sp("foo.drv"), 1, Some(0)).await; + insert_step(&mut conn, 1, 1, &sp("foo.drv")).await; + insert_output(&mut conn, 1, 1, "out", &sp("result-v1")).await; + insert_build_row(&mut conn, 2, &sp("foo.drv"), 1, Some(0)).await; + insert_step(&mut conn, 2, 1, &sp("foo.drv")).await; + insert_output(&mut conn, 2, 1, "out", &sp("result-v2")).await; + insert_build_row(&mut conn, 3, &sp("foo.drv"), 1, Some(0)).await; + + let r = conn.get_finished_build(3).await.unwrap().unwrap(); + assert_eq!( + r.outputs, + vec![("out".to_owned(), Some(store_str(&sp("result-v2"))))], + ); + } + + #[tokio::test] + async fn finished_build_ids_filters_to_finished() { + let (_pg, mut conn) = setup().await; + insert_build_row(&mut conn, 1, &sp("a.drv"), 0, None).await; + insert_build_row(&mut conn, 2, &sp("b.drv"), 1, Some(0)).await; + insert_build_row(&mut conn, 3, &sp("c.drv"), 1, Some(1)).await; + + let mut got = conn.finished_build_ids(&[1, 2, 3, 999]).await.unwrap(); + got.sort_unstable(); + // Any terminal status is enough for waiter wakeup; result mapping happens later. + assert_eq!(got, vec![2, 3]); + } + + #[tokio::test] + async fn finished_build_ids_handles_empty_input() { + let (_pg, mut conn) = setup().await; + let got = conn.finished_build_ids(&[]).await.unwrap(); + assert!(got.is_empty()); + } + + #[tokio::test] + async fn ensure_adhoc_jobset_is_idempotent() { + let (_pg, mut conn) = setup().await; + let id1 = conn.ensure_adhoc_jobset().await.unwrap(); + let id2 = conn.ensure_adhoc_jobset().await.unwrap(); + assert_eq!(id1, id2); + } + + #[tokio::test] + async fn insert_adhoc_build_sets_keep_and_starts_unfinished() { + let (_pg, mut conn) = setup().await; + let jobset_id = conn.ensure_adhoc_jobset().await.unwrap(); + let mut tx = conn.begin_transaction().await.unwrap(); + let build_id = tx + .insert_adhoc_build(jobset_id, "hello", "/nix/store/foo.drv", "x86_64-linux") + .await + .unwrap(); + tx.commit().await.unwrap(); + + let row: (i32, i32, String, String) = + sqlx::query_as("SELECT keep, finished, drvPath, system FROM Builds WHERE id = $1") + .bind(build_id) + .fetch_one(&mut *conn.conn) + .await + .unwrap(); + assert_eq!(row.0, 1, "keep=1 so hydra-update-gc-roots retains outputs"); + assert_eq!(row.1, 0, "build starts unfinished"); + assert_eq!(row.2, "/nix/store/foo.drv"); + assert_eq!(row.3, "x86_64-linux"); + } } diff --git a/subprojects/crates/db/src/lib.rs b/subprojects/crates/db/src/lib.rs index 7bac16acd..6aeda93f2 100644 --- a/subprojects/crates/db/src/lib.rs +++ b/subprojects/crates/db/src/lib.rs @@ -18,7 +18,7 @@ pub mod models; use std::str::FromStr as _; -pub use connection::{Connection, Transaction}; +pub use connection::{Connection, FinishedBuild, Transaction}; pub use harmonia_store_core::store_path::StoreDir; pub use sqlx::Error; @@ -37,6 +37,23 @@ impl Database { }) } + pub async fn new_with_options( + options: sqlx::postgres::PgConnectOptions, + max_connections: u32, + ) -> Result { + Ok(Self { + pool: sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .connect_with(options) + .await?, + }) + } + + #[must_use] + pub fn pool(&self) -> &sqlx::PgPool { + &self.pool + } + pub async fn get(&self) -> Result { let conn = self.pool.acquire().await?; Ok(Connection::new(conn)) diff --git a/subprojects/hydra-builder/package.nix b/subprojects/hydra-builder/package.nix index 4b4dde346..fdd52d13e 100644 --- a/subprojects/hydra-builder/package.nix +++ b/subprojects/hydra-builder/package.nix @@ -34,15 +34,15 @@ rustPlatform.buildRustPackage { cargoLock = { lockFile = ../../Cargo.lock; outputHashes = { - "harmonia-store-core-0.0.0-alpha.0" = "sha256-g7JJGrjWnnzBtxtxLaqL/wKehPBZAHh8C7U7ALYW6o0="; + "harmonia-store-core-0.0.0-alpha.0" = "sha256-EwOfW4esHMOaxoxgrguLJYLPQXoFjzOljR2+x+mmo3k="; }; }; - # The source fileset above intentionally excludes hydra-queue-runner, - # so drop it from the workspace members to keep cargo from trying to - # load its (absent) manifest. + # Drop the other Rust binary crates from the workspace; their sources + # are excluded from the fileset above, so cargo would otherwise fail + # trying to load their (absent) manifests. postPatch = '' - sed -i 's|"subprojects/hydra-queue-runner", ||' Cargo.toml + sed -i '/hydra-builder/!{/"subprojects\/hydra-/d;}' Cargo.toml ''; buildAndTestSubdir = "subprojects/hydra-builder"; diff --git a/subprojects/hydra-drv-daemon/Cargo.toml b/subprojects/hydra-drv-daemon/Cargo.toml new file mode 100644 index 000000000..40e4a3b3c --- /dev/null +++ b/subprojects/hydra-drv-daemon/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "hydra-drv-daemon" +version.workspace = true +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +anyhow.workspace = true +clap = { workspace = true, features = [ "derive", "env" ] } +fs-err.workspace = true +futures.workspace = true +tokio = { workspace = true, features = [ "full" ] } +tracing.workspace = true +tracing-subscriber = { workspace = true, features = [ "env-filter" ] } + +db = { path = "../crates/db" } +harmonia-protocol.workspace = true +harmonia-store-core.workspace = true +harmonia-store-remote.workspace = true +harmonia-utils-hash.workspace = true +harmonia-utils-io.workspace = true +sqlx = { workspace = true, features = [ "runtime-tokio", "postgres" ] } +thiserror.workspace = true diff --git a/subprojects/hydra-drv-daemon/package.nix b/subprojects/hydra-drv-daemon/package.nix new file mode 100644 index 000000000..dce660b3f --- /dev/null +++ b/subprojects/hydra-drv-daemon/package.nix @@ -0,0 +1,51 @@ +{ + lib, + version, + + rustPlatform, + + pkg-config, +}: + +rustPlatform.buildRustPackage { + pname = "hydra-drv-daemon"; + inherit version; + + src = lib.fileset.toSource { + root = ../..; + fileset = lib.fileset.unions [ + ../../Cargo.toml + ../../Cargo.lock + ../../.cargo + ../../.sqlx + ../../subprojects/hydra-drv-daemon/Cargo.toml + ../../subprojects/hydra-drv-daemon/src + ../../subprojects/crates + ../../subprojects/hydra/sql/hydra.sql + ../../subprojects/proto + ]; + }; + + cargoLock = { + lockFile = ../../Cargo.lock; + outputHashes = { + "harmonia-store-core-0.0.0-alpha.0" = "sha256-EwOfW4esHMOaxoxgrguLJYLPQXoFjzOljR2+x+mmo3k="; + }; + }; + + # Strip workspace siblings that aren't included in the source fileset + # so cargo doesn't try to load their manifests. + postPatch = '' + sed -i '/hydra-drv-daemon/!{/"subprojects\/hydra-/d;}' Cargo.toml + ''; + + buildAndTestSubdir = "subprojects/hydra-drv-daemon"; + + nativeBuildInputs = [ + pkg-config + ]; + + doCheck = false; + + meta.description = "Hydra drv-daemon: spawn ad-hoc Hydra builds via the nix daemon protocol"; +} diff --git a/subprojects/hydra-drv-daemon/src/handler.rs b/subprojects/hydra-drv-daemon/src/handler.rs new file mode 100644 index 000000000..ce8006bb1 --- /dev/null +++ b/subprojects/hydra-drv-daemon/src/handler.rs @@ -0,0 +1,763 @@ +use std::collections::BTreeMap; +use std::future::{Future, ready}; +use std::pin::Pin; + +use tokio::io::AsyncBufRead; + +use harmonia_protocol::build_result::{ + BuildResult, BuildResultFailure, BuildResultInner, BuildResultSuccess, FailureStatus, + SuccessStatus, +}; +use harmonia_protocol::daemon::{ + DaemonError as ProtocolError, DaemonResult, DaemonStore, FutureResultExt, HandshakeDaemonStore, + ResultLog, ResultLogExt, TrustLevel, +}; +use harmonia_protocol::daemon_wire::types2::{BuildMode, KeyedBuildResult}; +use harmonia_protocol::types::AddToStoreItem; +use harmonia_protocol::valid_path_info::{UnkeyedValidPathInfo, ValidPathInfo}; +use harmonia_store_core::derivation::{BasicDerivation, DerivationOutput}; +use harmonia_store_core::derived_path::{DerivedPath, OutputName, SingleDerivedPath}; +use harmonia_store_core::realisation::{DrvOutput, Realisation}; +use harmonia_store_core::store_path::{ + ContentAddressMethodAlgorithm, StorePath, StorePathHash, StorePathSet, +}; +use harmonia_store_remote::pool::{ConnectionPool, PoolConfig}; +use harmonia_utils_hash::Sha256; + +use db::StoreDir; +use db::models::BuildStatus; + +use crate::waiter::BuildWaiter; + +/// Daemon-store implementation that schedules build requests through Hydra. +#[derive(Clone)] +pub struct DrvDaemonHandler { + store_dir: StoreDir, + db: db::Database, + upstream: ConnectionPool, + waiter: BuildWaiter, +} + +impl std::fmt::Debug for DrvDaemonHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DrvDaemonHandler") + .field("store_dir", &self.store_dir) + .finish_non_exhaustive() + } +} + +impl DrvDaemonHandler { + pub fn new( + store_dir: StoreDir, + db: db::Database, + upstream_socket: &str, + waiter: BuildWaiter, + ) -> Self { + let upstream = ConnectionPool::with_store_dir( + upstream_socket, + store_dir.clone(), + PoolConfig::default(), + ); + Self { + store_dir, + db, + upstream, + waiter, + } + } + + /// Reject output maps with unresolved paths before synthesizing realisations. + async fn assert_static_outputs(&self, drv_path: &StorePath) -> Result<(), ProtocolError> { + let mut guard = self + .upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + let map = guard.client().query_derivation_output_map(drv_path).await?; + let unresolved: Vec = map + .iter() + .filter(|(_, path)| path.is_none()) + .map(|(name, _)| name.as_ref().to_owned()) + .collect(); + if !unresolved.is_empty() { + return Err(ProtocolError::custom(format!( + "drv-daemon BuildPathsWithResults requires static output paths; \ + {}/{drv_path} has unresolved (likely CA-floating) outputs: {}", + self.store_dir, + unresolved.join(", ") + ))); + } + Ok(()) + } + + /// Reject BuildDerivation requests whose .drv is not present for the queue runner to read. + async fn assert_drv_uploaded(&self, drv_path: &StorePath) -> Result<(), ProtocolError> { + let mut guard = self + .upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + if !guard.client().is_valid_path(drv_path).await? { + return Err(ProtocolError::custom(format!( + "drv-daemon BuildDerivation: {}/{drv_path} is not present in the \ + upstream store; upload it via add_to_store_nar / \ + add_multiple_to_store before requesting a build", + self.store_dir + ))); + } + Ok(()) + } + + /// Register the waiter before commit so the queue runner cannot finish the row first. + async fn run_adhoc_build( + &self, + drv_path: &str, + nix_name: &str, + system: &str, + ) -> Result { + let mut conn = self + .db + .get() + .await + .map_err(|e| ProtocolError::custom(format!("hydra db: {e}")))?; + let jobset_id = conn + .ensure_adhoc_jobset() + .await + .map_err(|e| ProtocolError::custom(format!("ensure adhoc jobset: {e}")))?; + let mut tx = conn + .begin_transaction() + .await + .map_err(|e| ProtocolError::custom(format!("begin tx: {e}")))?; + let build_id = tx + .insert_adhoc_build(jobset_id, nix_name, drv_path, system) + .await + .map_err(|e| ProtocolError::custom(format!("insert adhoc build: {e}")))?; + + let rx = self + .waiter + .register(build_id) + .await + .map_err(|e| ProtocolError::custom(format!("cannot schedule ad-hoc build: {e}")))?; + + // Keep waiter cleanup centralized for every pre-completion error path. + let result: Result = async { + tx.notify_builds_added() + .await + .map_err(|e| ProtocolError::custom(format!("notify builds_added: {e}")))?; + tx.commit() + .await + .map_err(|e| ProtocolError::custom(format!("commit: {e}")))?; + + tracing::info!( + build_id, + drv_path, + "scheduled ad-hoc build, awaiting finish" + ); + + if rx.await.is_err() { + return Err(ProtocolError::custom( + "build_finished listener went unhealthy mid-flight; \ + try again once the daemon reconnects", + )); + } + + let mut conn = self + .db + .get() + .await + .map_err(|e| ProtocolError::custom(format!("hydra db: {e}")))?; + conn.get_finished_build(build_id) + .await + .map_err(|e| ProtocolError::custom(format!("read finished build: {e}")))? + .ok_or_else(|| { + ProtocolError::custom(format!("build {build_id} woke but has no finished row")) + }) + } + .await; + + if result.is_err() { + self.waiter.forget(build_id).await; + } + result + } +} + +/// Reject non-IA derivations because the daemon cannot return real CA realisations. +fn require_input_addressed( + drv_path: &str, + outputs: impl IntoIterator, &'static str)>, +) -> Result<(), ProtocolError> { + let mut bad = Vec::new(); + for (name, kind) in outputs { + if kind != "InputAddressed" { + bad.push(format!("{}={kind}", name.as_ref())); + } + } + if !bad.is_empty() { + return Err(ProtocolError::custom(format!( + "drv-daemon only supports input-addressed derivations; \ + {drv_path} has non-IA outputs: {}", + bad.join(", ") + ))); + } + Ok(()) +} + +fn output_kind(o: &DerivationOutput) -> &'static str { + match o { + DerivationOutput::InputAddressed(_) => "InputAddressed", + DerivationOutput::CAFixed(_) => "CAFixed", + DerivationOutput::CAFloating(_) => "CAFloating", + DerivationOutput::Deferred => "Deferred", + DerivationOutput::Impure(_) => "Impure", + } +} + +fn require_normal_mode(mode: BuildMode) -> Result<(), ProtocolError> { + if mode == BuildMode::Normal { + Ok(()) + } else { + Err(ProtocolError::custom(format!( + "drv-daemon only supports BuildMode::Normal, got {mode:?}" + ))) + } +} + +fn drv_path_of(path: &SingleDerivedPath) -> &StorePath { + let mut current = path; + loop { + match current { + SingleDerivedPath::Opaque(sp) => return sp, + SingleDerivedPath::Built { drv_path, .. } => current = drv_path.as_ref(), + } + } +} + +fn finished_to_build_result( + store_dir: &StoreDir, + drv_path: &str, + finished: &db::FinishedBuild, +) -> Result { + let inner = match finished.status { + BuildStatus::Success => BuildResultInner::Success(BuildResultSuccess { + status: SuccessStatus::Built, + built_outputs: synthesize_built_outputs(store_dir, drv_path, &finished.outputs)?, + }), + status => BuildResultInner::Failure(BuildResultFailure { + status: build_status_to_failure(status), + error_msg: build_status_message(status).into(), + is_non_deterministic: status == BuildStatus::NotDeterministic, + }), + }; + Ok(BuildResult { + inner, + times_built: 1, + start_time: finished.start_time.unwrap_or(0).into(), + stop_time: finished.stop_time.unwrap_or(0).into(), + cpu_user: None, + cpu_system: None, + }) +} + +/// Synthesize realisations from recorded output paths; missing paths are queue-runner bugs. +fn synthesize_built_outputs( + store_dir: &StoreDir, + drv_path: &str, + outputs: &[(String, Option)], +) -> Result, ProtocolError> { + let drv_hash = Sha256::digest(drv_path.as_bytes()).into(); + let mut map = BTreeMap::new(); + for (name, path) in outputs { + let Some(path) = path else { + return Err(ProtocolError::custom(format!( + "ad-hoc build of {drv_path} succeeded but BuildStepOutputs.path \ + for '{name}' is NULL — daemon refuses to fabricate a realisation \ + from a missing path" + ))); + }; + let output_name: OutputName = name + .parse() + .map_err(|e| ProtocolError::custom(format!("invalid output name {name:?}: {e}")))?; + let out_path = store_dir + .parse(path) + .map_err(|e| ProtocolError::custom(format!("invalid output path {path:?}: {e}")))?; + map.insert( + output_name.clone(), + Realisation { + id: DrvOutput { + drv_hash, + output_name, + }, + out_path, + signatures: Default::default(), + dependent_realisations: Default::default(), + }, + ); + } + Ok(map) +} + +fn build_status_to_failure(status: BuildStatus) -> FailureStatus { + match status { + BuildStatus::Failed | BuildStatus::FailedWithOutput => FailureStatus::PermanentFailure, + BuildStatus::DepFailed => FailureStatus::DependencyFailed, + BuildStatus::TimedOut => FailureStatus::TimedOut, + BuildStatus::CachedFailure => FailureStatus::CachedFailure, + BuildStatus::LogLimitExceeded => FailureStatus::LogLimitExceeded, + BuildStatus::NarSizeLimitExceeded => FailureStatus::OutputRejected, + BuildStatus::NotDeterministic => FailureStatus::NotDeterministic, + BuildStatus::Unsupported + | BuildStatus::Aborted + | BuildStatus::Cancelled + | BuildStatus::Busy + | BuildStatus::Resolved + | BuildStatus::Success => FailureStatus::MiscFailure, + } +} + +fn build_status_message(status: BuildStatus) -> &'static str { + match status { + BuildStatus::Success => "succeeded", + BuildStatus::Failed => "build failed", + BuildStatus::DepFailed => "a dependency failed to build", + BuildStatus::Aborted => "build was aborted", + BuildStatus::Cancelled => "build was cancelled", + BuildStatus::FailedWithOutput => "build failed (with output)", + BuildStatus::TimedOut => "build timed out", + BuildStatus::CachedFailure => "build previously failed (cached)", + BuildStatus::Unsupported => "system not supported", + BuildStatus::LogLimitExceeded => "log size limit exceeded", + BuildStatus::NarSizeLimitExceeded => "NAR size limit exceeded", + BuildStatus::NotDeterministic => "build is not deterministic", + BuildStatus::Busy => "build is still in progress", + BuildStatus::Resolved => "CA derivation resolved (transient state)", + } +} + +impl HandshakeDaemonStore for DrvDaemonHandler { + type Store = Self; + + fn handshake(self) -> impl ResultLog> + Send { + ready(Ok(self)).empty_logs() + } +} + +impl DaemonStore for DrvDaemonHandler { + fn trust_level(&self) -> TrustLevel { + TrustLevel::Trusted + } + + fn set_options<'a>( + &'a mut self, + _options: &'a harmonia_protocol::types::ClientOptions, + ) -> impl ResultLog> + Send + 'a { + // Client options are intentionally ignored; proxied reads use upstream's settings. + ready(Ok(())).empty_logs() + } + + fn build_derivation<'a>( + &'a mut self, + drv_path: &'a StorePath, + drv: &'a BasicDerivation, + mode: BuildMode, + ) -> impl ResultLog> + Send + 'a { + let this = self.clone(); + async move { + require_normal_mode(mode)?; + require_input_addressed( + &format!("{}/{}", this.store_dir, drv_path), + drv.outputs + .iter() + .map(|(n, o)| (n.as_ref(), output_kind(o))), + )?; + this.assert_drv_uploaded(drv_path).await?; + let drv_path_str = format!("{}/{}", this.store_dir, drv_path); + let nix_name: String = drv.name.to_string(); + let system = std::str::from_utf8(&drv.platform) + .map_err(|e| ProtocolError::custom(format!("non-utf8 platform: {e}")))?; + let finished = this + .run_adhoc_build(&drv_path_str, &nix_name, system) + .await?; + finished_to_build_result(&this.store_dir, &drv_path_str, &finished) + } + .empty_logs() + } + + fn build_paths<'a>( + &'a mut self, + drvs: &'a [DerivedPath], + mode: BuildMode, + ) -> impl ResultLog> + Send + 'a { + let this = self.clone(); + async move { + require_normal_mode(mode)?; + for path in drvs { + let drv_path = match path { + DerivedPath::Built { drv_path, .. } => drv_path_of(drv_path), + DerivedPath::Opaque(_) => { + return Err(ProtocolError::custom( + "drv-daemon BuildPaths does not support opaque (already-built) paths", + )); + } + }; + let drv_path_str = format!("{}/{}", this.store_dir, drv_path); + let nix_name = drv_path.name().as_ref().to_owned(); + let finished = this.run_adhoc_build(&drv_path_str, &nix_name, "").await?; + if finished.status != BuildStatus::Success { + return Err(ProtocolError::custom(format!( + "ad-hoc build of {drv_path_str} failed: {}", + build_status_message(finished.status) + ))); + } + } + Ok(()) + } + .empty_logs() + } + + fn build_paths_with_results<'a>( + &'a mut self, + drvs: &'a [DerivedPath], + mode: BuildMode, + ) -> impl ResultLog>> + Send + 'a { + let this = self.clone(); + async move { + require_normal_mode(mode)?; + let mut results = Vec::with_capacity(drvs.len()); + for path in drvs { + let drv_path = match path { + DerivedPath::Built { drv_path, .. } => drv_path_of(drv_path), + DerivedPath::Opaque(_) => { + return Err(ProtocolError::custom( + "drv-daemon BuildPathsWithResults does not support opaque paths", + )); + } + }; + this.assert_static_outputs(drv_path).await?; + let drv_path_str = format!("{}/{}", this.store_dir, drv_path); + let nix_name = drv_path.name().as_ref().to_owned(); + let finished = this.run_adhoc_build(&drv_path_str, &nix_name, "").await?; + let result = finished_to_build_result(&this.store_dir, &drv_path_str, &finished)?; + results.push(KeyedBuildResult { + path: path.clone(), + result, + }); + } + Ok(results) + } + .empty_logs() + } + + fn is_valid_path<'a>( + &'a mut self, + path: &'a StorePath, + ) -> impl ResultLog> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().is_valid_path(path).await + } + .empty_logs() + } + + fn query_valid_paths<'a>( + &'a mut self, + paths: &'a StorePathSet, + substitute: bool, + ) -> impl ResultLog> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_valid_paths(paths, substitute).await + } + .empty_logs() + } + + fn query_path_info<'a>( + &'a mut self, + path: &'a StorePath, + ) -> impl ResultLog>> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_path_info(path).await + } + .empty_logs() + } + + fn query_path_from_hash_part<'a>( + &'a mut self, + hash: &'a StorePathHash, + ) -> impl ResultLog>> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_path_from_hash_part(hash).await + } + .empty_logs() + } + + fn query_derivation_output_map<'a>( + &'a mut self, + path: &'a StorePath, + ) -> impl ResultLog< + Output = DaemonResult< + std::collections::BTreeMap< + harmonia_store_core::derived_path::OutputName, + Option, + >, + >, + > + Send + + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_derivation_output_map(path).await + } + .empty_logs() + } + + fn query_missing<'a>( + &'a mut self, + paths: &'a [DerivedPath], + ) -> impl ResultLog< + Output = DaemonResult, + > + Send + + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_missing(paths).await + } + .empty_logs() + } + + fn query_referrers<'a>( + &'a mut self, + path: &'a StorePath, + ) -> impl ResultLog> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_referrers(path).await + } + .empty_logs() + } + + fn query_substitutable_paths<'a>( + &'a mut self, + paths: &'a StorePathSet, + ) -> impl ResultLog> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_substitutable_paths(paths).await + } + .empty_logs() + } + + fn query_valid_derivers<'a>( + &'a mut self, + path: &'a StorePath, + ) -> impl ResultLog> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_valid_derivers(path).await + } + .empty_logs() + } + + fn query_realisation<'a>( + &'a mut self, + output_id: &'a harmonia_store_core::realisation::DrvOutput, + ) -> impl ResultLog< + Output = DaemonResult< + std::collections::BTreeSet, + >, + > + Send + + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().query_realisation(output_id).await + } + .empty_logs() + } + + fn add_temp_root<'a>( + &'a mut self, + path: &'a StorePath, + ) -> impl ResultLog> + Send + 'a { + // Temp roots attach to the upstream pooled connection, so they may outlive this client. + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().add_temp_root(path).await + } + .empty_logs() + } + + fn ensure_path<'a>( + &'a mut self, + path: &'a StorePath, + ) -> impl ResultLog> + Send + 'a { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().ensure_path(path).await + } + .empty_logs() + } + + fn add_indirect_root<'a>( + &'a mut self, + _path: &'a harmonia_protocol::types::DaemonPath, + ) -> impl ResultLog> + Send + 'a { + // Upstream proxying cannot decode this op's log payload yet; return unsupported rather than pretending a root was created. + ready(Err(ProtocolError::custom( + "drv-daemon does not support AddIndirectRoot; \ + use `nix-store --add-root ` without `--indirect` \ + (the daemon proxies AddPermRoot to the upstream nix-daemon)", + ))) + .empty_logs() + } + + fn add_perm_root<'a>( + &'a mut self, + store_path: &'a StorePath, + gc_root: &'a harmonia_protocol::types::DaemonPath, + ) -> impl ResultLog> + Send + 'a + { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard.client().add_perm_root(store_path, gc_root).await + } + .empty_logs() + } + + fn add_ca_to_store<'a, 'r, R>( + &'a mut self, + name: &'a str, + cam: ContentAddressMethodAlgorithm, + refs: &'a StorePathSet, + repair: bool, + source: R, + ) -> Pin> + Send + 'r>> + where + R: AsyncBufRead + Send + Unpin + 'r, + 'a: 'r, + { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard + .client() + .add_ca_to_store(name, cam, refs, repair, source) + .await + } + .empty_logs() + .boxed_result() + } + + fn add_multiple_to_store<'s, 'i, 'r, S, R>( + &'s mut self, + repair: bool, + dont_check_sigs: bool, + stream: S, + ) -> Pin> + Send + 'r>> + where + S: futures::Stream, ProtocolError>> + Send + 'i, + R: AsyncBufRead + Send + Unpin + 'i, + 's: 'r, + 'i: 'r, + { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard + .client() + .add_multiple_to_store(repair, dont_check_sigs, stream) + .await + } + .empty_logs() + .boxed_result() + } + + fn add_to_store_nar<'s, 'r, 'i, R>( + &'s mut self, + info: &'i ValidPathInfo, + source: R, + repair: bool, + dont_check_sigs: bool, + ) -> Pin> + Send + 'r>> + where + R: AsyncBufRead + Send + Unpin + 'r, + 's: 'r, + 'i: 'r, + { + let upstream = self.upstream.clone(); + async move { + let mut guard = upstream + .acquire() + .await + .map_err(|e| ProtocolError::custom(format!("upstream pool: {e}")))?; + guard + .client() + .add_to_store_nar(info, source, repair, dont_check_sigs) + .await + } + .empty_logs() + .boxed_result() + } + + fn shutdown(&mut self) -> impl Future> + Send + '_ { + ready(Ok(())) + } +} diff --git a/subprojects/hydra-drv-daemon/src/main.rs b/subprojects/hydra-drv-daemon/src/main.rs new file mode 100644 index 000000000..44cc92f8b --- /dev/null +++ b/subprojects/hydra-drv-daemon/src/main.rs @@ -0,0 +1,48 @@ +mod handler; +mod server; +mod waiter; + +use std::path::PathBuf; + +use clap::Parser; +use harmonia_store_core::store_path::StoreDir; + +#[derive(Parser, Debug)] +#[command(about = "Nix daemon proxy that writes derivations to Postgres")] +struct Args { + /// Unix socket to listen on for nix daemon connections. + #[arg(long, default_value = "/tmp/hydra-drv-daemon.sock")] + socket: PathBuf, + + /// Upstream nix daemon socket to proxy read operations to. + #[arg(long, default_value = "/nix/var/nix/daemon-socket/socket")] + upstream_socket: String, + + /// PostgreSQL connection URL. + #[arg(long, env = "HYDRA_DBA")] + db_url: String, + + /// Nix store directory. + #[arg(long, default_value = "/nix/store")] + store_dir: String, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), + ) + .init(); + + let args = Args::parse(); + let store_dir = + StoreDir::new(&args.store_dir).map_err(|e| anyhow::anyhow!("invalid store dir: {e}"))?; + let database = db::Database::new(&args.db_url, 4).await?; + let waiter = waiter::BuildWaiter::start(&database).await?; + let handler = + handler::DrvDaemonHandler::new(store_dir.clone(), database, &args.upstream_socket, waiter); + let server = server::DaemonServer::new(handler, args.socket, store_dir); + server.serve().await?; + Ok(()) +} diff --git a/subprojects/hydra-drv-daemon/src/server.rs b/subprojects/hydra-drv-daemon/src/server.rs new file mode 100644 index 000000000..736e6653e --- /dev/null +++ b/subprojects/hydra-drv-daemon/src/server.rs @@ -0,0 +1,480 @@ +// Nix daemon server, using harmonia-protocol's wire types. +// Derived from harmonia-daemon::server (EUPL-1.2 OR MIT). + +use std::fmt::Debug; +use std::path::PathBuf; +use std::pin::pin; + +use futures::{FutureExt, StreamExt as _}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, copy_buf}; +use tokio::net::UnixListener; +use tracing::{debug, error, info}; + +use harmonia_protocol::ProtocolVersion; +use harmonia_protocol::daemon::{ + DaemonError, DaemonResult, DaemonStore, HandshakeDaemonStore, ResultLog, + wire::{ + CLIENT_MAGIC, FramedReader, IgnoredOne, SERVER_MAGIC, + logger::RawLogMessage, + parse_add_multiple_to_store, + types2::{BaseStorePath, Request}, + }, +}; +use harmonia_protocol::de::{NixRead, NixReader}; +use harmonia_protocol::log::LogMessage; +use harmonia_protocol::ser::{NixWrite, NixWriter}; +use harmonia_store_core::store_path::StoreDir; +use harmonia_utils_io::{AsyncBufReadCompat, BytesReader}; + +const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::from_parts(1, 37); +const NIX_VERSION: &str = "2.24.0 (Hydra)"; + +struct RecoverableError { + can_recover: bool, + source: DaemonError, +} + +impl> From for RecoverableError { + fn from(source: T) -> Self { + RecoverableError { + can_recover: false, + source: source.into(), + } + } +} + +trait RecoverExt { + fn recover(self) -> Result; +} + +impl> RecoverExt for Result { + fn recover(self) -> Result { + self.map_err(|source| RecoverableError { + can_recover: true, + source: source.into(), + }) + } +} + +fn log_to_raw(msg: LogMessage) -> RawLogMessage { + match msg { + LogMessage::Message(m) => RawLogMessage::Next(m.text), + LogMessage::StartActivity(a) => RawLogMessage::StartActivity(a), + LogMessage::StopActivity(a) => RawLogMessage::StopActivity(a), + LogMessage::Result(r) => RawLogMessage::Result(r), + } +} + +async fn write_log(writer: &mut NixWriter, msg: LogMessage) -> Result<(), RecoverableError> +where + W: AsyncWrite + Send + Unpin, +{ + writer.write_value(&log_to_raw(msg)).await?; + Ok(()) +} + +async fn process_logs<'s, T: Send + 's, W>( + writer: &mut NixWriter, + logs: impl ResultLog> + Send + 's, +) -> Result +where + W: AsyncWrite + Send + Unpin, +{ + let mut logs = pin!(logs); + while let Some(msg) = logs.next().await { + write_log(writer, msg).await?; + } + logs.await.recover() +} + +struct DaemonConnection { + reader: NixReader>, + writer: NixWriter, +} + +impl DaemonConnection +where + R: AsyncRead + Debug + Send + Unpin, + W: AsyncWrite + Debug + Send + Unpin, +{ + async fn handshake(&mut self) -> DaemonResult<()> { + let magic: u64 = self.reader.read_number().await?; + if magic != CLIENT_MAGIC { + return Err(DaemonError::custom(format!("bad client magic: {magic:#x}"))); + } + self.writer.write_number(SERVER_MAGIC).await?; + self.writer.write_value(&PROTOCOL_VERSION).await?; + self.writer.flush().await?; + + let client_version: ProtocolVersion = self.reader.read_value().await?; + self.reader.set_version(client_version); + self.writer.set_version(client_version); + + if client_version.minor() >= 14 { + let _: bool = self.reader.read_value().await?; + } + if client_version.minor() >= 11 { + let _: bool = self.reader.read_value().await?; + } + if client_version.minor() >= 33 { + self.writer.write_value(NIX_VERSION).await?; + } + if client_version.minor() >= 35 { + self.writer + .write_value(&harmonia_protocol::types::TrustLevel::Trusted) + .await?; + } + self.writer.flush().await?; + Ok(()) + } + + async fn process_logs_write<'s, T: Send + 's>( + &'s mut self, + logs: impl ResultLog> + Send + 's, + ) -> Result { + let value = process_logs(&mut self.writer, logs).await?; + self.writer.write_value(&RawLogMessage::Last).await?; + Ok(value) + } + + async fn process_requests<'s, S>(&'s mut self, mut store: S) -> Result<(), DaemonError> + where + S: DaemonStore + 's, + { + loop { + let fut = self.reader.try_read_value::().boxed(); + let res = fut.await?; + let Some(request) = res else { + break; + }; + let op = request.operation(); + debug!("got op {}", op); + if let Err(mut err) = self.process_request(&mut store, request).await { + error!(error = ?err.source, recover=err.can_recover, "error processing request"); + err.source = err.source.fill_operation(op); + if err.can_recover { + self.writer + .write_value(&RawLogMessage::Error(err.source.into())) + .await?; + } else { + return Err(err.source); + } + } + self.writer.flush().await?; + } + store.shutdown().await + } + + async fn process_request<'s, S>( + &'s mut self, + store: &mut S, + request: Request, + ) -> Result<(), RecoverableError> + where + S: DaemonStore + 's, + { + use Request::*; + match request { + SetOptions(options) => { + let logs = store.set_options(&options); + self.process_logs_write(logs).await?; + } + IsValidPath(path) => { + let logs = store.is_valid_path(&path); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + QueryValidPaths(req) => { + let logs = store.query_valid_paths(&req.paths, req.substitute); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + QueryPathInfo(path) => { + let logs = store.query_path_info(&path); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + NarFromPath(path) => { + let logs = store.nar_from_path(&path); + let mut logs = pin!(logs); + while let Some(msg) = logs.next().await { + write_log(&mut self.writer, msg).await?; + } + let mut reader = pin!(logs.await?); + self.writer.write_value(&RawLogMessage::Last).await?; + copy_buf(&mut reader, &mut self.writer) + .await + .map_err(DaemonError::from)?; + } + QueryReferrers(path) => { + let logs = store.query_referrers(&path); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + AddToStore(req) => { + let buf_reader = AsyncBufReadCompat::new(&mut self.reader); + let mut framed = FramedReader::new(buf_reader); + let logs = + store.add_ca_to_store(&req.name, req.cam, &req.refs, req.repair, &mut framed); + let res = process_logs(&mut self.writer, logs).await; + let err = framed.drain_all().await; + let value = res?; + err?; + self.writer.write_value(&RawLogMessage::Last).await?; + self.writer.write_value(&value).await?; + } + BuildPaths(req) => { + let logs = store.build_paths(&req.paths, req.mode); + self.process_logs_write(logs).await?; + self.writer.write_value(&IgnoredOne).await?; + } + EnsurePath(path) => { + let logs = store.ensure_path(&path); + self.process_logs_write(logs).await?; + self.writer.write_value(&IgnoredOne).await?; + } + AddTempRoot(path) => { + let logs = store.add_temp_root(&path); + self.process_logs_write(logs).await?; + self.writer.write_value(&IgnoredOne).await?; + } + AddIndirectRoot(path) => { + let logs = store.add_indirect_root(&path); + self.process_logs_write(logs).await?; + self.writer.write_value(&IgnoredOne).await?; + } + FindRoots => { + let logs = store.find_roots(); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + CollectGarbage(req) => { + let logs = store.collect_garbage( + req.action, + &req.paths_to_delete, + req.ignore_liveness, + req.max_freed, + ); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + QueryAllValidPaths => { + let logs = store.query_all_valid_paths(); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + QueryPathFromHashPart(hash) => { + let logs = store.query_path_from_hash_part(&hash); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + QuerySubstitutablePaths(paths) => { + let logs = store.query_substitutable_paths(&paths); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + QueryValidDerivers(path) => { + let logs = store.query_valid_derivers(&path); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + OptimiseStore => { + let logs = store.optimise_store(); + self.process_logs_write(logs).await?; + self.writer.write_value(&IgnoredOne).await?; + } + VerifyStore(req) => { + let logs = store.verify_store(req.check_contents, req.repair); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + BuildDerivation(req) => { + let (drv_path, drv) = &req.drv; + let logs = store.build_derivation(drv_path, drv, req.mode); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + AddSignatures(req) => { + let logs = store.add_signatures(&req.path, &req.signatures); + self.process_logs_write(logs).await?; + self.writer.write_value(&IgnoredOne).await?; + } + AddToStoreNar(req) => { + let buf_reader = AsyncBufReadCompat::new(&mut self.reader); + let mut framed = FramedReader::new(buf_reader); + let logs = store.add_to_store_nar( + &req.path_info, + &mut framed, + req.repair, + req.dont_check_sigs, + ); + let res: Result<(), RecoverableError> = async { + let mut logs = pin!(logs); + while let Some(msg) = logs.next().await { + write_log(&mut self.writer, msg).await?; + } + logs.await.recover()?; + Ok(()) + } + .await; + let err = framed.drain_all().await; + res?; + err?; + self.writer.write_value(&RawLogMessage::Last).await?; + } + QueryMissing(paths) => { + let logs = store.query_missing(&paths); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + QueryDerivationOutputMap(path) => { + let logs = store.query_derivation_output_map(&path); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + RegisterDrvOutput(realisation) => { + let logs = store.register_drv_output(&realisation); + self.process_logs_write(logs).await?; + } + QueryRealisation(output_id) => { + let logs = store.query_realisation(&output_id); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + AddMultipleToStore(req) => { + let builder = NixReader::builder().set_version(self.reader.version()); + let buf_reader = AsyncBufReadCompat::new(&mut self.reader); + let mut framed = FramedReader::new(buf_reader); + let source = builder.build_buffered(&mut framed); + let stream = parse_add_multiple_to_store(source).await?; + let logs = store.add_multiple_to_store(req.repair, req.dont_check_sigs, stream); + let res: Result<(), RecoverableError> = async { + let mut logs = pin!(logs); + while let Some(msg) = logs.next().await { + write_log(&mut self.writer, msg).await?; + } + logs.await.recover()?; + self.writer.write_value(&RawLogMessage::Last).await?; + Ok(()) + } + .await; + let err = framed.drain_all().await; + res?; + err?; + } + AddBuildLog(BaseStorePath(path)) => { + let buf_reader = AsyncBufReadCompat::new(&mut self.reader); + let mut framed = FramedReader::new(buf_reader); + let logs = store.add_build_log(&path, &mut framed); + let res = process_logs(&mut self.writer, logs).await; + let err = framed.drain_all().await; + res?; + err?; + self.writer.write_value(&RawLogMessage::Last).await?; + self.writer.write_value(&IgnoredOne).await?; + } + BuildPathsWithResults(req) => { + let logs = store.build_paths_with_results(&req.paths, req.mode); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + AddPermRoot(req) => { + let logs = store.add_perm_root(&req.store_path, &req.gc_root); + let value = self.process_logs_write(logs).await?; + self.writer.write_value(&value).await?; + } + } + Ok(()) + } +} + +#[derive(Debug)] +pub struct DaemonServer { + handler: H, + socket_path: PathBuf, + store_dir: StoreDir, +} + +impl DaemonServer +where + H: HandshakeDaemonStore + Clone + Send + Sync + 'static, +{ + pub fn new(handler: H, socket_path: PathBuf, store_dir: StoreDir) -> Self { + Self { + handler, + socket_path, + store_dir, + } + } + + pub async fn serve(&self) -> Result<(), std::io::Error> { + if self.socket_path.exists() { + fs_err::remove_file(&self.socket_path)?; + } + if let Some(parent) = self.socket_path.parent() { + fs_err::create_dir_all(parent)?; + } + + let listener = UnixListener::bind(&self.socket_path)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + // 0660 allows hydra-group clients without opening ad-hoc build submission to all local users. + let perms = std::fs::Permissions::from_mode(0o660); + fs_err::set_permissions(&self.socket_path, perms)?; + } + + info!("nix daemon listening on {:?}", self.socket_path); + + loop { + let (stream, _addr) = listener.accept().await?; + let handler = self.handler.clone(); + let store_dir = self.store_dir.clone(); + + tokio::spawn(async move { + let (reader, writer) = stream.into_split(); + let reader = NixReader::builder() + .set_store_dir(&store_dir) + .build_buffered(reader); + let writer = NixWriter::builder().set_store_dir(&store_dir).build(writer); + let mut conn = DaemonConnection { reader, writer }; + + if let Err(e) = conn.handshake().await { + error!("handshake error: {e:?}"); + return; + } + + let store = { + let logs = handler.handshake(); + let mut logs = std::pin::pin!(logs); + while let Some(msg) = futures::StreamExt::next(&mut logs).await { + if let Err(e) = conn.writer.write_value(&log_to_raw(msg)).await { + error!("handshake log error: {e:?}"); + return; + } + } + if let Err(e) = conn.writer.write_value(&RawLogMessage::Last).await { + error!("handshake terminator error: {e:?}"); + return; + } + match logs.await { + Ok(s) => s, + Err(e) => { + error!("store handshake error: {e:?}"); + return; + } + } + }; + if let Err(e) = conn.writer.flush().await { + error!("handshake flush error: {e:?}"); + return; + } + + if let Err(e) = conn.process_requests(store).await { + error!("connection error: {e:?}"); + } + }); + } + } +} diff --git a/subprojects/hydra-drv-daemon/src/waiter.rs b/subprojects/hydra-drv-daemon/src/waiter.rs new file mode 100644 index 000000000..9f0cccea6 --- /dev/null +++ b/subprojects/hydra-drv-daemon/src/waiter.rs @@ -0,0 +1,235 @@ +//! Async wakeup for ad-hoc Builds finished by the Hydra queue runner. +//! +//! The daemon listens on the `build_finished` postgres channel and +//! dispatches each notification to the waiting handler that registered +//! the build id. +//! +//! Reliability rules: +//! +//! * If the listener loses its connection, the registry of waiters is +//! *kept*: a waiter that registered while healthy survives transient +//! reconnects. The task retries with exponential backoff. The strike +//! counter is shared between "listener construction failed" and +//! "listener stream errored mid-flight" so a flaky construct -> +//! immediate-error loop still trips the exit threshold. +//! +//! * After reconnect, finished registered builds are swept from the DB +//! before new registrations are allowed; PostgreSQL does not replay +//! missed notifications. +//! +//! * While the listener is unhealthy, `register` returns +//! `RegisterError::Unhealthy` so handlers can fail their request +//! instead of inserting a *new* row that can race with the gap. +//! +//! * After `MAX_RECONNECT_ATTEMPTS` consecutive strikes the task +//! drains all pending waiters and `std::process::exit(1)`s so a +//! service supervisor can restart the daemon with a clean slate. +//! Drained waiters yield `Err(RecvError)` and their handlers +//! surface a build-failure response. + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use futures::StreamExt as _; +use tokio::sync::{Mutex, oneshot}; + +use db::models::BuildID; + +const MAX_RECONNECT_ATTEMPTS: u32 = 5; +const INITIAL_BACKOFF: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(30); + +type WaiterMap = Arc>>>; + +#[derive(Debug, thiserror::Error)] +pub enum RegisterError { + #[error("build_finished listener is currently unhealthy; refusing to register a waiter")] + Unhealthy, +} + +/// Registry of in-flight ad-hoc builds. Cloning shares the same backing +/// state, so all daemon connections wake from the same listener task. +#[derive(Clone)] +pub struct BuildWaiter { + waiters: WaiterMap, + healthy: Arc, +} + +impl std::fmt::Debug for BuildWaiter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BuildWaiter").finish_non_exhaustive() + } +} + +impl BuildWaiter { + pub async fn start(db: &db::Database) -> Result { + // Verify LISTEN works before accepting registrations. + drop(db.listener(vec!["build_finished"]).await?); + + let waiters: WaiterMap = Arc::new(Mutex::new(HashMap::new())); + let healthy = Arc::new(AtomicBool::new(true)); + tokio::spawn(run_listener(db.clone(), waiters.clone(), healthy.clone())); + + Ok(Self { waiters, healthy }) + } + + /// Register a build while the listener is healthy; existing waiters survive reconnect and may be woken by the sweep. + pub async fn register( + &self, + build_id: BuildID, + ) -> Result, RegisterError> { + if !self.healthy.load(Ordering::Acquire) { + return Err(RegisterError::Unhealthy); + } + let (tx, rx) = oneshot::channel(); + self.waiters.lock().await.insert(build_id, tx); + Ok(rx) + } + + /// Drop a pending registration without waking it. Used to clean up + /// when a caller bails out before the build finishes. + pub async fn forget(&self, build_id: BuildID) { + self.waiters.lock().await.remove(&build_id); + } +} + +async fn run_listener(db: db::Database, waiters: WaiterMap, healthy: Arc) { + let mut strike = 0u32; + loop { + let stream = match db.listener(vec!["build_finished"]).await { + Ok(s) => s, + Err(e) => { + strike += 1; + healthy.store(false, Ordering::Release); + tracing::error!( + strike, + max_strikes = MAX_RECONNECT_ATTEMPTS, + "build_finished listener reconnect failed: {e}" + ); + if strike >= MAX_RECONNECT_ATTEMPTS { + tracing::error!( + "giving up on build_finished listener; exiting so a supervisor can restart" + ); + drain_waiters(&waiters).await; + std::process::exit(1); + } + tokio::time::sleep(backoff_for(strike)).await; + continue; + } + }; + + // Sweep after LISTEN because PostgreSQL does not replay missed notifications. + if let Err(e) = sweep_already_finished(&db, &waiters).await { + // A listener without a working DB query cannot safely become healthy. + strike += 1; + tracing::error!( + strike, + max_strikes = MAX_RECONNECT_ATTEMPTS, + "post-reconnect sweep failed: {e}" + ); + if strike >= MAX_RECONNECT_ATTEMPTS { + tracing::error!("giving up on build_finished listener; exiting"); + drain_waiters(&waiters).await; + std::process::exit(1); + } + tokio::time::sleep(backoff_for(strike)).await; + continue; + } + + healthy.store(true, Ordering::Release); + + let mut stream = Box::pin(stream); + let mut got_notification = false; + loop { + match stream.next().await { + Some(Ok(notif)) => { + got_notification = true; + dispatch(¬if, &waiters).await; + } + Some(Err(e)) => { + tracing::error!("build_finished listener error: {e}"); + break; + } + None => { + tracing::warn!("build_finished listener stream ended unexpectedly"); + break; + } + } + } + + // Keep registered waiters across the LISTEN gap; reset strikes only after a useful notification. + healthy.store(false, Ordering::Release); + if got_notification { + strike = 0; + } else { + strike += 1; + if strike >= MAX_RECONNECT_ATTEMPTS { + tracing::error!("listener kept failing without delivering anything; exiting"); + drain_waiters(&waiters).await; + std::process::exit(1); + } + tokio::time::sleep(backoff_for(strike)).await; + } + } +} + +async fn drain_waiters(waiters: &WaiterMap) { + let mut map = waiters.lock().await; + let count = map.len(); + if count > 0 { + tracing::warn!(count, "dropping pending build_finished waiters before exit"); + // Dropping senders wakes receivers with RecvError. + map.clear(); + } +} + +async fn sweep_already_finished(db: &db::Database, waiters: &WaiterMap) -> Result<(), db::Error> { + let registered: Vec = { + let map = waiters.lock().await; + map.keys().copied().collect() + }; + if registered.is_empty() { + return Ok(()); + } + + let mut conn = db.get().await?; + let finished = conn.finished_build_ids(®istered).await?; + if finished.is_empty() { + return Ok(()); + } + + tracing::info!( + count = finished.len(), + "waking waiters for builds that finished during the listener gap" + ); + + let mut map = waiters.lock().await; + for id in finished { + if let Some(tx) = map.remove(&id) { + let _ = tx.send(()); + } + } + Ok(()) +} + +async fn dispatch(notif: &sqlx::postgres::PgNotification, waiters: &WaiterMap) { + let payload = notif.payload(); + let mut map = waiters.lock().await; + for id_str in payload.split('\t') { + let Ok(id) = id_str.parse::() else { + continue; + }; + if let Some(tx) = map.remove(&id) { + let _ = tx.send(()); + } + } +} + +fn backoff_for(strike: u32) -> Duration { + let exp = INITIAL_BACKOFF + .checked_mul(2u32.saturating_pow(strike.saturating_sub(1))) + .unwrap_or(MAX_BACKOFF); + exp.min(MAX_BACKOFF) +} diff --git a/subprojects/hydra-evaluator/Cargo.toml b/subprojects/hydra-evaluator/Cargo.toml new file mode 100644 index 000000000..48e101aec --- /dev/null +++ b/subprojects/hydra-evaluator/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "hydra-evaluator" +version.workspace = true +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +anyhow.workspace = true +clap = { workspace = true, features = [ "derive" ] } +fs-err.workspace = true +futures.workspace = true +sqlx = { workspace = true, features = [ "runtime-tokio", "tls-rustls-ring-webpki", "postgres" ] } +tokio = { workspace = true, features = [ "full" ] } +tokio-stream.workspace = true +tracing.workspace = true + +db = { path = "../crates/db" } +hydra-tracing = { path = "../crates/tracing" } diff --git a/subprojects/hydra-evaluator/package.nix b/subprojects/hydra-evaluator/package.nix new file mode 100644 index 000000000..3321f096e --- /dev/null +++ b/subprojects/hydra-evaluator/package.nix @@ -0,0 +1,64 @@ +{ + lib, + version, + + rustPlatform, + + nixComponents, + protobuf, + pkg-config, + rust-jemalloc-sys, +}: + +rustPlatform.buildRustPackage { + pname = "hydra-evaluator"; + inherit version; + + src = lib.fileset.toSource { + root = ../..; + fileset = lib.fileset.unions [ + ../../Cargo.toml + ../../Cargo.lock + ../../.cargo + ../../.sqlx + ../../subprojects/hydra-evaluator/Cargo.toml + ../../subprojects/hydra-evaluator/src + ../../subprojects/crates + # For unit tests which want to spin up a fresh database + ../../subprojects/hydra/sql/hydra.sql + ../../subprojects/proto + ]; + }; + + cargoLock = { + lockFile = ../../Cargo.lock; + outputHashes = { + "harmonia-store-core-0.0.0-alpha.0" = "sha256-g7JJGrjWnnzBtxtxLaqL/wKehPBZAHh8C7U7ALYW6o0="; + }; + }; + + # Drop the other Rust binary crates from the workspace; their sources + # are excluded from the fileset above, so cargo would otherwise fail + # trying to load their (absent) manifests. + postPatch = '' + sed -i '/hydra-evaluator/!{/"subprojects\/hydra-/d;}' Cargo.toml + ''; + + buildAndTestSubdir = "subprojects/hydra-evaluator"; + + nativeBuildInputs = [ + pkg-config + protobuf + ]; + + buildInputs = [ + nixComponents.nix-main + protobuf + rust-jemalloc-sys + ]; + + # FIXME: get these passing in a prod build + doCheck = false; + + meta.description = "Hydra evaluator (Rust)"; +} diff --git a/subprojects/hydra-evaluator/src/config.rs b/subprojects/hydra-evaluator/src/config.rs new file mode 100644 index 000000000..af43ceab1 --- /dev/null +++ b/subprojects/hydra-evaluator/src/config.rs @@ -0,0 +1,137 @@ +use std::collections::HashMap; + +use anyhow::Context as _; +use sqlx::postgres::{PgConnectOptions, PgSslMode}; + +#[derive(Debug)] +pub(crate) struct HydraConfig { + options: HashMap, +} + +impl HydraConfig { + pub(crate) fn load() -> Self { + let mut options = HashMap::new(); + + let path = match std::env::var("HYDRA_CONFIG") { + Ok(p) if !p.is_empty() => p, + _ => return Self { options }, + }; + + let contents = match fs_err::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + tracing::warn!("could not read HYDRA_CONFIG at {path}: {e}"); + return Self { options }; + } + }; + + for line in contents.lines() { + // Strip comments + let line = match line.find('#') { + Some(pos) => &line[..pos], + None => line, + }; + let line = line.trim(); + + let Some(eq) = line.find('=') else { + continue; + }; + + let key = line[..eq].trim(); + let value = line[eq + 1..].trim(); + + if key.is_empty() { + continue; + } + + options.insert(key.to_owned(), value.to_owned()); + } + + Self { options } + } + + pub(crate) fn get_int(&self, key: &str, default: u64) -> u64 { + self.options + .get(key) + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + } +} + +/// Parse a `HYDRA_DBI` environment variable into `PgConnectOptions`. +/// +/// Accepts strings like `dbi:Pg:dbname=hydra;host=localhost;port=5432`. +pub(crate) fn parse_hydra_dbi() -> anyhow::Result { + let dbi = std::env::var("HYDRA_DBI").unwrap_or_else(|_| "dbi:Pg:dbname=hydra;".to_owned()); + parse_dbi(&dbi) +} + +fn parse_dbi(dbi: &str) -> anyhow::Result { + let params = dbi + .strip_prefix("dbi:Pg:") + .or_else(|| dbi.strip_prefix("DBI:Pg:")) + .context("$HYDRA_DBI does not denote a PostgreSQL database")?; + + let mut opts = PgConnectOptions::new(); + + for pair in params.split(';').filter(|s| !s.is_empty()) { + let (key, value) = pair + .split_once('=') + .with_context(|| format!("invalid DBI parameter: {pair}"))?; + match key.trim() { + "dbname" => opts = opts.database(value.trim()), + "host" => opts = opts.host(value.trim()), + "port" => { + opts = opts.port( + value + .trim() + .parse() + .with_context(|| format!("invalid port: {value}"))?, + ); + } + "user" => opts = opts.username(value.trim()), + "password" => opts = opts.password(value.trim()), + "application_name" => opts = opts.application_name(value.trim()), + // The C++ evaluator used libpq which understood these + // natively; without explicit handling TLS-required + // deployments would fail. + "sslmode" => { + let mode = match value.trim() { + "disable" => PgSslMode::Disable, + "allow" => PgSslMode::Allow, + "prefer" => PgSslMode::Prefer, + "require" => PgSslMode::Require, + "verify-ca" => PgSslMode::VerifyCa, + "verify-full" => PgSslMode::VerifyFull, + v => anyhow::bail!("invalid sslmode: {v}"), + }; + opts = opts.ssl_mode(mode); + } + "sslrootcert" => opts = opts.ssl_root_cert(value.trim()), + "sslcert" => opts = opts.ssl_client_cert(value.trim()), + "sslkey" => opts = opts.ssl_client_key(value.trim()), + // Warn rather than bail on unknown parameters, to avoid + // breaking on libpq keywords we haven't mapped yet. + other => { + tracing::warn!("ignoring unsupported DBI parameter: {other}"); + } + } + } + + Ok(opts) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_simple_dbi() { + parse_dbi("dbi:Pg:dbname=hydra;host=localhost;port=5432").unwrap(); + } + + #[test] + fn parse_dbi_uppercase() { + parse_dbi("DBI:Pg:dbname=testdb;").unwrap(); + } +} diff --git a/subprojects/hydra-evaluator/src/evaluator.rs b/subprojects/hydra-evaluator/src/evaluator.rs new file mode 100644 index 000000000..75f7e5b82 --- /dev/null +++ b/subprojects/hydra-evaluator/src/evaluator.rs @@ -0,0 +1,608 @@ +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context as _; +use futures::StreamExt as _; +use tokio::process::Command; +use tokio::sync::{Mutex, Notify}; + +use crate::config::HydraConfig; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +enum EvaluationStyle { + Schedule = 1, + Oneshot = 2, + OneAtATime = 3, +} + +impl EvaluationStyle { + fn from_i32(v: i32) -> Option { + match v { + 1 => Some(Self::Schedule), + 2 => Some(Self::Oneshot), + 3 => Some(Self::OneAtATime), + _ => None, + } + } +} + +#[derive(Debug)] +struct Jobset { + id: i32, + project: String, + name: String, + evaluation_style: Option, + last_checked_time: i64, + trigger_time: Option, + check_interval: i64, +} + +impl Jobset { + fn display(&self) -> String { + format!("{}:{} (jobset#{})", self.project, self.name, self.id) + } +} + +#[derive(Debug, Default)] +struct State { + running_evals: usize, + running_ids: HashSet, + jobsets: BTreeMap, +} + +pub(crate) struct Evaluator { + db: db::Database, + max_evals: usize, + eval_one: Option<(String, String)>, + state: Arc>, + notify_work: Arc, +} + +impl Evaluator { + pub(crate) fn new( + db: db::Database, + config: &HydraConfig, + eval_one: Option<(String, String)>, + ) -> Self { + let max_evals = + usize::try_from(config.get_int("max_concurrent_evals", 4).max(1)).unwrap_or(usize::MAX); + Self { + db, + max_evals, + eval_one, + state: Arc::new(Mutex::new(State::default())), + notify_work: Arc::new(Notify::new()), + } + } + + pub(crate) async fn run(self) -> anyhow::Result<()> { + self.unlock().await?; + + let this = Arc::new(self); + + let monitor = { + let this = Arc::clone(&this); + tokio::spawn(async move { + this.db_monitor_task().await; + }) + }; + + let main_loop = { + let this = Arc::clone(&this); + tokio::spawn(async move { + this.main_loop_task().await; + }) + }; + + let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?; + let mut sigterm = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + + tokio::select! { + _ = sigint.recv() => { + tracing::info!("received SIGINT, exiting"); + std::process::exit(1); + } + _ = sigterm.recv() => { + tracing::info!("received SIGTERM, exiting"); + std::process::exit(1); + } + _ = monitor => { + tracing::error!("database monitor exited unexpectedly"); + std::process::exit(1); + } + _ = main_loop => { + tracing::error!("main loop exited unexpectedly"); + std::process::exit(1); + } + } + } + + async fn main_loop_task(&self) { + loop { + if let Err(e) = self.main_loop_iteration().await { + tracing::error!("exception in main loop: {e:#}"); + tokio::time::sleep(Duration::from_secs(30)).await; + } + } + } + + async fn main_loop_iteration(&self) -> anyhow::Result<()> { + loop { + let sleep_duration = { + let state = self.state.lock().await; + self.compute_sleep_duration(&state) + }; + + tracing::debug!("waiting for {} s", sleep_duration.as_secs()); + + if sleep_duration == Duration::MAX { + self.notify_work.notified().await; + } else { + tokio::select! { + () = tokio::time::sleep(sleep_duration) => {} + () = self.notify_work.notified() => {} + } + } + + let mut state = self.state.lock().await; + self.start_evals(&mut state).await?; + } + } + + fn compute_sleep_duration(&self, state: &State) -> Duration { + if state.running_evals >= self.max_evals { + return Duration::MAX; + } + + let now = now_epoch(); + let mut sleep_secs = i64::MAX; + + for jobset in state.jobsets.values() { + if state.running_ids.contains(&jobset.id) { + continue; + } + if jobset.check_interval > 0 { + let next = jobset.last_checked_time + jobset.check_interval - now; + sleep_secs = sleep_secs.min(next.max(1)); + } + } + + u64::try_from(sleep_secs).map_or(Duration::MAX, Duration::from_secs) + } + + async fn db_monitor_task(&self) { + loop { + if let Err(e) = self.db_monitor_loop().await { + tracing::error!("exception in database monitor: {e:#}"); + if is_broken_connection(&e) { + tracing::error!("database connection broken, exiting"); + std::process::exit(1); + } + tokio::time::sleep(Duration::from_secs(30)).await; + } + } + } + + async fn db_monitor_loop(&self) -> anyhow::Result<()> { + let mut stream = self + .db + .listener(vec![ + "jobsets_added", + "jobsets_deleted", + "jobset_scheduling_changed", + ]) + .await?; + + // Read initial state before waiting for notifications + self.read_jobsets().await?; + self.notify_work.notify_one(); + + loop { + match stream.next().await { + Some(Ok(_notification)) => { + tracing::info!("received jobset event"); + self.read_jobsets().await?; + self.notify_work.notify_one(); + } + Some(Err(e)) => { + return Err(e.into()); + } + None => { + anyhow::bail!("notification stream ended"); + } + } + } + } + + async fn read_jobsets(&self) -> anyhow::Result<()> { + let rows = sqlx::query_as::<_, JobsetRow>( + "SELECT j.id, project, j.name, \ + lastCheckedTime, triggerTime, checkInterval, \ + j.enabled as jobset_enabled \ + FROM Jobsets j \ + JOIN Projects p ON j.project = p.name \ + WHERE j.enabled != 0 AND p.enabled != 0", + ) + .fetch_all(self.db.pool()) + .await?; + + let mut state = self.state.lock().await; + let mut seen = HashSet::new(); + + for row in &rows { + if let Some((ref proj, ref js)) = self.eval_one + && (row.project != *proj || row.name != *js) + { + continue; + } + + seen.insert(row.id); + + let evaluation_style = EvaluationStyle::from_i32(row.jobset_enabled); + + let jobset = state.jobsets.entry(row.id).or_insert_with(|| Jobset { + id: row.id, + project: row.project.clone(), + name: row.name.clone(), + evaluation_style: None, + last_checked_time: 0, + trigger_time: None, + check_interval: 0, + }); + + jobset.project.clone_from(&row.project); + jobset.name.clone_from(&row.name); + jobset.last_checked_time = row.lastcheckedtime.map_or(0, i64::from); + jobset.trigger_time = row.triggertime.map(i64::from); + jobset.check_interval = row.checkinterval.into(); + jobset.evaluation_style = evaluation_style; + } + + if self.eval_one.is_some() && seen.is_empty() { + tracing::error!("the specified jobset does not exist or is disabled"); + std::process::exit(1); + } + + state.jobsets.retain(|id, jobset| { + if seen.contains(id) { + true + } else { + tracing::info!("forgetting jobset '{}'", jobset.display()); + false + } + }); + + Ok(()) + } + + async fn should_evaluate(&self, jobset: &Jobset, state: &State) -> bool { + if state.running_ids.contains(&jobset.id) { + tracing::debug!("shouldEvaluate {}? no: already running", jobset.display()); + return false; + } + + if jobset.trigger_time.is_some() { + tracing::debug!("shouldEvaluate {}? yes: requested", jobset.display()); + return true; + } + + if jobset.check_interval <= 0 { + tracing::debug!( + "shouldEvaluate {}? no: checkInterval <= 0", + jobset.display() + ); + return false; + } + + let now = now_epoch(); + if jobset.last_checked_time + jobset.check_interval <= now { + if jobset.evaluation_style == Some(EvaluationStyle::OneAtATime) { + return self.should_evaluate_one_at_a_time(jobset).await; + } + tracing::debug!( + "shouldEvaluate(oneshot/scheduled) {}? yes: checkInterval elapsed", + jobset.display() + ); + return true; + } + + false + } + + async fn should_evaluate_one_at_a_time(&self, jobset: &Jobset) -> bool { + let eval_id: Option = match sqlx::query_scalar( + "SELECT id FROM JobsetEvals WHERE jobset_id = $1 ORDER BY id DESC LIMIT 1", + ) + .bind(jobset.id) + .fetch_optional(self.db.pool()) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("error checking one-at-a-time for {}: {e}", jobset.display()); + return false; + } + }; + + let Some(eval_id) = eval_id else { + tracing::debug!( + "shouldEvaluate(one-at-a-time) {}? yes: no prior eval", + jobset.display() + ); + return true; + }; + + let unfinished: Option = match sqlx::query_scalar( + "SELECT id FROM Builds \ + JOIN JobsetEvalMembers ON (JobsetEvalMembers.build = Builds.id) \ + WHERE JobsetEvalMembers.eval = $1 AND builds.finished = 0 \ + LIMIT 1", + ) + .bind(eval_id) + .fetch_optional(self.db.pool()) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!( + "error checking unfinished builds for {}: {e}", + jobset.display() + ); + return false; + } + }; + + if unfinished.is_none() { + tracing::debug!( + "shouldEvaluate(one-at-a-time) {}? yes: no unfinished builds", + jobset.display() + ); + true + } else { + tracing::debug!( + "shouldEvaluate(one-at-a-time) {}? no: at least one unfinished build", + jobset.display() + ); + false + } + } + + async fn start_evals(&self, state: &mut State) -> anyhow::Result<()> { + // Collect eligible jobset IDs with their sort keys + let mut candidates: Vec<(i32, Option, i64)> = Vec::new(); + + for jobset in state.jobsets.values() { + if self.eval_one.is_some() + || (jobset.evaluation_style.is_some() && self.should_evaluate(jobset, state).await) + { + candidates.push((jobset.id, jobset.trigger_time, jobset.last_checked_time)); + } + } + + // Sort by (trigger_time, last_checked_time, id) + // None trigger_time sorts after Some (not triggered = lowest priority) + candidates.sort_by(|a, b| { + let ta = a.1.unwrap_or(i64::MAX); + let tb = b.1.unwrap_or(i64::MAX); + ta.cmp(&tb).then(a.2.cmp(&b.2)).then(a.0.cmp(&b.0)) + }); + + for (jobset_id, _, _) in candidates { + if state.running_evals >= self.max_evals { + break; + } + // Re-borrow the jobset from state + if let Some(jobset) = state.jobsets.get(&jobset_id) { + self.start_eval( + state, + jobset_id, + jobset.project.clone(), + jobset.name.clone(), + ) + .await?; + } + } + + Ok(()) + } + + async fn start_eval( + &self, + state: &mut State, + jobset_id: i32, + project: String, + jobset_name: String, + ) -> anyhow::Result<()> { + let now = now_epoch(); + let jobset_display = format!("{project}:{jobset_name} (jobset#{jobset_id})"); + + let last_checked = state + .jobsets + .get(&jobset_id) + .map_or(0, |j| j.last_checked_time); + + tracing::info!( + "starting evaluation of jobset '{}' (last checked {} s ago)", + jobset_display, + now - last_checked, + ); + + sqlx::query("UPDATE Jobsets SET startTime = $1 WHERE id = $2") + .bind(now_epoch_i32()) + .bind(jobset_id) + .execute(self.db.pool()) + .await + .context("failed to set startTime")?; + + let child = Command::new("hydra-eval-jobset") + .arg(&project) + .arg(&jobset_name) + .spawn() + .with_context(|| format!("failed to spawn hydra-eval-jobset for {jobset_display}"))?; + + state.running_evals += 1; + state.running_ids.insert(jobset_id); + + let eval_one = self.eval_one.is_some(); + let db = self.db.clone(); + let state_arc = Arc::clone(&self.state); + let notify_work = Arc::clone(&self.notify_work); + + tokio::spawn(async move { + reap_child( + child, + jobset_id, + jobset_display, + eval_one, + db, + state_arc, + notify_work, + ) + .await; + }); + + Ok(()) + } + + async fn unlock(&self) -> anyhow::Result<()> { + sqlx::query("UPDATE Jobsets SET startTime = null") + .execute(self.db.pool()) + .await + .context("failed to unlock jobsets")?; + Ok(()) + } +} + +async fn reap_child( + mut child: tokio::process::Child, + jobset_id: i32, + jobset_display: String, + eval_one: bool, + db: db::Database, + state: Arc>, + notify_work: Arc, +) { + let status = child.wait().await; + + let (exit_ok, status_str) = match &status { + Ok(s) => { + let code = s.code(); + let ok = code == Some(0) || code == Some(1); + (ok, format!("{s}")) + } + Err(e) => (false, format!("error: {e}")), + }; + + tracing::info!("evaluation of jobset '{}' {}", jobset_display, status_str); + + let now = now_epoch(); + + { + let mut st = state.lock().await; + if st.running_evals > 0 { + st.running_evals -= 1; + } + st.running_ids.remove(&jobset_id); + + if let Some(jobset) = st.jobsets.get_mut(&jobset_id) { + jobset.trigger_time = None; + jobset.last_checked_time = now; + } + } + + if let Err(e) = update_db_after_eval(&db, jobset_id, exit_ok, &status_str, now).await { + tracing::error!("exception setting jobset error: {e:#}"); + } + + notify_work.notify_one(); + + if eval_one { + std::process::exit(0); + } +} + +async fn update_db_after_eval( + db: &db::Database, + jobset_id: i32, + exit_ok: bool, + status_str: &str, + now: i64, +) -> anyhow::Result<()> { + // Wrap in a transaction (as the C++ original did) so that a partial + // failure cannot clear triggerTime but leave startTime set, which + // would make the jobset appear permanently running. + let mut txn = db.pool().begin().await?; + + // Clear trigger time to prevent stuck eval loop + sqlx::query( + "UPDATE Jobsets SET triggerTime = null \ + WHERE id = $1 AND startTime IS NOT NULL AND triggerTime <= startTime", + ) + .bind(jobset_id) + .execute(&mut *txn) + .await?; + + // Clear start time + sqlx::query("UPDATE Jobsets SET startTime = null WHERE id = $1") + .bind(jobset_id) + .execute(&mut *txn) + .await?; + + if !exit_ok { + sqlx::query( + "UPDATE Jobsets SET errorMsg = $1, lastCheckedTime = $2, \ + errorTime = $2, fetchErrorMsg = null WHERE id = $3", + ) + .bind(format!("evaluation {status_str}")) + .bind(i32::try_from(now).unwrap_or(i32::MAX)) + .bind(jobset_id) + .execute(&mut *txn) + .await?; + } + + txn.commit().await?; + Ok(()) +} + +fn now_epoch() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|d| i64::try_from(d.as_secs()).ok()) + .unwrap_or(0) +} + +/// Current epoch narrowed to the INT4 width used by Hydra's schema. +/// Saturates at `i32::MAX` (the schema has a pre-existing Y2038 limit). +fn now_epoch_i32() -> i32 { + i32::try_from(now_epoch()).unwrap_or(i32::MAX) +} + +fn is_broken_connection(e: &anyhow::Error) -> bool { + for cause in e.chain() { + if let Some(sqlx_err) = cause.downcast_ref::() + && matches!(sqlx_err, sqlx::Error::Io(_) | sqlx::Error::PoolClosed) + { + return true; + } + } + false +} + +/// Row type for the Jobsets query. `lastcheckedtime` and `triggertime` +/// are INT4 in the Postgres schema; `sqlx` refuses to decode INT4 into +/// `i64`, so these must be `i32`. +#[derive(sqlx::FromRow)] +#[allow(non_snake_case)] +struct JobsetRow { + id: i32, + project: String, + name: String, + lastcheckedtime: Option, + triggertime: Option, + checkinterval: i32, + jobset_enabled: i32, +} diff --git a/subprojects/hydra-evaluator/src/main.rs b/subprojects/hydra-evaluator/src/main.rs new file mode 100644 index 000000000..bfa9d10ae --- /dev/null +++ b/subprojects/hydra-evaluator/src/main.rs @@ -0,0 +1,68 @@ +#![forbid(unsafe_code)] +#![deny( + clippy::all, + clippy::pedantic, + clippy::expect_used, + clippy::unwrap_used, + future_incompatible, + missing_debug_implementations, + nonstandard_style, + unreachable_pub, + missing_copy_implementations, + unused_qualifications +)] +#![allow(clippy::missing_errors_doc)] + +mod config; +mod evaluator; + +use anyhow::Context as _; +use clap::Parser; + +use config::{HydraConfig, parse_hydra_dbi}; +use evaluator::Evaluator; + +#[derive(clap::Parser, Debug)] +#[command(name = "hydra-evaluator", about = "Hydra jobset evaluation scheduler")] +struct Cli { + /// Clear startTime on all jobsets and exit + #[arg(long)] + unlock: bool, + + /// Project name (requires jobset) + project: Option, + + /// Jobset name (requires project) + jobset: Option, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let _tracing_guard = hydra_tracing::init()?; + let cli = Cli::parse(); + + let db_opts = parse_hydra_dbi()?; + let db = db::Database::new_with_options(db_opts, 4) + .await + .context("failed to connect to database")?; + + if cli.unlock { + sqlx::query("UPDATE Jobsets SET startTime = null") + .execute(db.pool()) + .await + .context("failed to unlock jobsets")?; + tracing::info!("unlocked all jobsets"); + return Ok(()); + } + + let config = HydraConfig::load(); + + let eval_one = match (cli.project, cli.jobset) { + (Some(p), Some(j)) => Some((p, j)), + (None, None) => None, + _ => anyhow::bail!("Syntax: hydra-evaluator [ ]"), + }; + + let evaluator = Evaluator::new(db, &config, eval_one); + evaluator.run().await +} diff --git a/subprojects/hydra-manual/src/configuration.md b/subprojects/hydra-manual/src/configuration.md index 491376b3e..ad97e6b77 100644 --- a/subprojects/hydra-manual/src/configuration.md +++ b/subprojects/hydra-manual/src/configuration.md @@ -102,6 +102,41 @@ See [`nix help stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html) for a description of the store URI format. +Routing import-from-derivation through Hydra +-------------------------------------------- + +When `allow_import_from_derivation = true` is set, an evaluator that +hits an import-from-derivation builds the inner derivation locally on +the evaluator host. Pointing the evaluator at the `hydra-drv-daemon` +socket via `NIX_REMOTE` instead causes those builds to land in the +`Builds` table under the auto-created `adhoc/adhoc` jobset, where the +queue runner picks them up like any other build. + +The wiring is environment-only: `hydra-evaluator` (and any tool +invoking the Nix daemon protocol) will use whatever socket +`NIX_REMOTE` points at. There is no Hydra-side configuration knob. + +NixOS users can opt in via the evaluator module: + +```nix +services.hydra-drv-daemon-dev.enable = true; +services.hydra-evaluator-dev.routeIfdsThroughDaemon = true; +services.hydra-evaluator-dev.allowImportFromDerivation = true; +``` + +This sets `NIX_REMOTE=unix:///run/hydra-drv-daemon/socket` on the +`hydra-evaluator.service` unit and adds `allow_import_from_derivation += true` to `hydra.conf`. Without `allowImportFromDerivation`, +`nix-eval-jobs` rejects every IFD before the daemon can see it. + +For non-NixOS deployments, run `hydra-drv-daemon` yourself and set +`NIX_REMOTE` in the environment of `hydra-evaluator` (and any other +process that should route store ops through the daemon). + +Content-addressed IFDs are not yet supported through this path — the +daemon does not surface output realisations, so the evaluator cannot +read CA outputs back. Input-addressed IFDs work end-to-end. + Statsd Configuration -------------------- diff --git a/subprojects/hydra-queue-runner/package.nix b/subprojects/hydra-queue-runner/package.nix index 3de5687bd..bb54e5513 100644 --- a/subprojects/hydra-queue-runner/package.nix +++ b/subprojects/hydra-queue-runner/package.nix @@ -36,15 +36,15 @@ rustPlatform.buildRustPackage { cargoLock = { lockFile = ../../Cargo.lock; outputHashes = { - "harmonia-store-core-0.0.0-alpha.0" = "sha256-g7JJGrjWnnzBtxtxLaqL/wKehPBZAHh8C7U7ALYW6o0="; + "harmonia-store-core-0.0.0-alpha.0" = "sha256-EwOfW4esHMOaxoxgrguLJYLPQXoFjzOljR2+x+mmo3k="; }; }; - # The source fileset above intentionally excludes hydra-builder, - # so drop it from the workspace members to keep cargo from trying to - # load its (absent) manifest. + # Drop the other Rust binary crates from the workspace; their sources + # are excluded from the fileset above, so cargo would otherwise fail + # trying to load their (absent) manifests. postPatch = '' - sed -i 's|"subprojects/hydra-builder", ||' Cargo.toml + sed -i '/hydra-queue-runner/!{/"subprojects\/hydra-/d;}' Cargo.toml ''; buildAndTestSubdir = "subprojects/hydra-queue-runner"; diff --git a/subprojects/hydra-tests/drv-daemon/adhoc.t b/subprojects/hydra-tests/drv-daemon/adhoc.t new file mode 100644 index 000000000..b6d9e5677 --- /dev/null +++ b/subprojects/hydra-tests/drv-daemon/adhoc.t @@ -0,0 +1,83 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; +use Test2::V0; +use DrvDaemonContext; +use Hydra::Helper::Exec; + +# Imperative ad-hoc build: nix-store --realise against the drv-daemon +# socket lands the build in the auto-created `adhoc/adhoc` jobset, the +# queue runner picks it up like any other Build, and the daemon +# returns once the row is finished. +# +# This path doesn't go through hydra-evaluator at all. + +my $ctx = test_context(); +my $jobsdir = $ctx->jobsdir; + +my $drv; +{ + local @ENV{keys %{$ctx->{central_env}}} = values %{$ctx->{central_env}}; + my ($res, $stdout, $stderr) = captureStdoutStderr(60, + "nix-instantiate", "$jobsdir/drv-daemon/adhoc.nix", "-A", "hello", + ); + if ($res) { + chomp $stderr; + diag("nix-instantiate failed: $stderr"); + die "nix-instantiate failed\n"; + } + chomp $stdout; + $stdout =~ s/!.*$//; + $drv = $stdout; +} + +ok($drv =~ m{\.drv$}, "instantiated to $drv"); + +my $stack = DrvDaemonContext->new($ctx); + +my ($res, $stdout, $stderr) = $stack->run_cmd(900, + "nix-store", "--realise", $drv, +); +if ($res) { + chomp $stderr; + diag("nix-store --realise failed: $stderr"); +} +$stack->pump_logs; +is(($res // 0) + 0, 0, "nix-store --realise via drv-daemon succeeds"); + +my @output_paths = grep { /\.drv$/ ? 0 : 1 } split /\n/, ($stdout // ""); +ok(scalar(@output_paths) >= 1, "nix-store printed at least one output path"); +for my $p (@output_paths) { + ok(-e $p, "output path exists: $p"); +} + +my $db = $ctx->db(); +my @builds = $db->resultset('Builds')->search( + { 'jobset.project' => 'adhoc', 'jobset.name' => 'adhoc' }, + { join => 'jobset', order_by => 'me.id desc' }, +); +ok(scalar(@builds) >= 1, "drv-daemon created an ad-hoc Builds row"); + +my $build = $builds[0]; +$build->discard_changes; +is($build->finished, 1, "ad-hoc build is marked finished"); +is($build->buildstatus, 0, "ad-hoc build succeeded"); +is($build->drvpath, $drv, "ad-hoc build drvpath matches the submitted derivation"); +is($build->keep, 1, "ad-hoc build is marked keep=1 so its outputs survive gc"); + +# hydra-update-gc-roots roots kept outputs via BuildOutputs. +my @outputs = $build->buildoutputs->all; +ok(scalar(@outputs) >= 1, "ad-hoc build has at least one BuildOutputs row"); +my %paths = map { $_->name => $_->path } @outputs; +ok(defined $paths{out}, "BuildOutputs row for 'out' has a path recorded"); +ok($paths{out} && -e $paths{out}, "recorded BuildOutputs path exists on disk"); + +# The daemon socket gates ad-hoc build submission, so it must not be world-writable. +my @sb = stat($stack->daemon_socket); +ok(scalar(@sb) > 0, "daemon socket is stat-able"); +is($sb[2] & 07777, 0660, "daemon socket is mode 0660 (group-rw, other-none)"); + +$stack->stop; + +done_testing; diff --git a/subprojects/hydra-tests/jobs/drv-daemon/adhoc.nix b/subprojects/hydra-tests/jobs/drv-daemon/adhoc.nix new file mode 100644 index 000000000..dd2cbc1d7 --- /dev/null +++ b/subprojects/hydra-tests/jobs/drv-daemon/adhoc.nix @@ -0,0 +1,18 @@ +# Plain input-addressed derivation submitted directly to hydra-drv-daemon +# via `nix-store --realise`, without any Hydra jobset / evaluator round-trip. +let + cfg = import ../config.nix; +in +{ + hello = cfg.mkDerivation { + name = "hello-adhoc"; + builder = "/bin/sh"; + args = [ + "-c" + '' + mkdir -p $out + echo "hello from drv-daemon" > $out/result + '' + ]; + }; +} diff --git a/subprojects/hydra-tests/lib/CliRunners.pm b/subprojects/hydra-tests/lib/CliRunners.pm index bb0cd52f7..d732fa72b 100644 --- a/subprojects/hydra-tests/lib/CliRunners.pm +++ b/subprojects/hydra-tests/lib/CliRunners.pm @@ -13,9 +13,21 @@ our @EXPORT = qw( sendNotifications ); +# When $ctx has a DrvDaemonContext attached (HYDRA_TEST_USE_DRV_DAEMON +# env), eval children get NIX_REMOTE pointed at the daemon socket so +# IFD store ops route through it. +sub _eval_env { + my ($ctx) = @_; + my %env = %{ $ctx->{central_env} }; + my $d = $ctx->drv_daemon; + $env{NIX_REMOTE} = $d->nix_remote_url if $d; + return %env; +} + sub evalSucceeds { my ($ctx, $jobset) = @_; - local @ENV{keys %{$ctx->{central_env}}} = values %{$ctx->{central_env}}; + my %env = _eval_env($ctx); + local @ENV{keys %env} = values %env; my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name)); $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB if ($res) { @@ -31,7 +43,8 @@ sub evalSucceeds { sub evalFails { my ($ctx, $jobset) = @_; - local @ENV{keys %{$ctx->{central_env}}} = values %{$ctx->{central_env}}; + my %env = _eval_env($ctx); + local @ENV{keys %env} = values %env; my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name)); $jobset->discard_changes({ '+columns' => {'errormsg' => 'errormsg'} }); # refresh from DB if (!$res) { diff --git a/subprojects/hydra-tests/lib/DrvDaemonContext.pm b/subprojects/hydra-tests/lib/DrvDaemonContext.pm new file mode 100644 index 000000000..eb0e176ce --- /dev/null +++ b/subprojects/hydra-tests/lib/DrvDaemonContext.pm @@ -0,0 +1,273 @@ +use warnings; +use strict; + +package DrvDaemonContext; + +# Spin up the full stack required for builds submitted directly through +# hydra-drv-daemon: an upstream nix-daemon (so the drv-daemon has +# something to proxy reads / .drv uploads to), the drv-daemon itself, +# and a queue-runner + builder pair (shared with QueueRunnerBuildOne +# via QueueRunnerContext) that picks up the rows it inserts. +# +# Returns an object whose DESTROY tears everything down. Use +# `daemon_socket` to point a `nix-build` / `nix-store` invocation at +# the drv-daemon, and `nix_remote_url` for the same over `NIX_REMOTE`. + +use IPC::Run; +use LWP::UserAgent; +use URI::Escape qw(uri_escape); +use QueueRunnerContext qw( + start_queue_runner + start_builder + wait_for_socket + wait_for_url +); + +our @ISA = qw(Exporter); +our @EXPORT = qw(); + +sub _flush_stream { + my ($label, $stream, $buf_ref, $final) = @_; + return if $$buf_ref eq ""; + utf8::decode($$buf_ref) or warn "Invalid unicode in $label $stream."; + while ($$buf_ref =~ s/^([^\n]*)\n//) { + print STDERR "[$label $stream] $1\n"; + } + if ($final && $$buf_ref ne "") { + print STDERR "[$label $stream] $$buf_ref\n"; + $$buf_ref = ""; + } +} + +sub _flush_proc { + my ($p, $final) = @_; + _flush_stream($p->{label}, "stdout", $p->{out}, $final); + _flush_stream($p->{label}, "stderr", $p->{err}, $final); +} + +sub new { + my ($class, $ctx) = @_; + ref $ctx eq 'HydraTestContext' + or die "DrvDaemonContext requires a HydraTestContext\n"; + + my $tmpdir = $ctx->{tmpdir}; + my $upstream_sock = "$tmpdir/upstream-nix-daemon.sock"; + my $daemon_sock = "$tmpdir/drv-daemon.sock"; + + my $self = bless { + ctx => $ctx, + upstream_sock => $upstream_sock, + daemon_sock => $daemon_sock, + procs => {}, + }, $class; + + $self->_spawn_upstream; + $self->_spawn_drv_daemon; + + my ($qr_harness, $base_url, $grpc_port, $qr_out, $qr_err) = + start_queue_runner($ctx, + queue_monitor_loop => 1, + rust_log => "queue_runner=debug,info", + ); + $self->{procs}{queue_runner} = { + label => "Queue runner", + harness => $qr_harness, + out => $qr_out, + err => $qr_err, + }; + $self->{base_url} = $base_url; + $self->{grpc_port} = $grpc_port; + $self->{ua} = LWP::UserAgent->new(timeout => 2); + + my ($bl_harness, $bl_out, $bl_err) = + start_builder($ctx, $grpc_port, rust_log => "hydra_builder=debug,info"); + $self->{procs}{builder} = { + label => "Builder", + harness => $bl_harness, + out => $bl_out, + err => $bl_err, + }; + + wait_for_url($self->{ua}, "$base_url/status/machines", sub { + shift->decoded_content =~ /"hostname"/; + }) or die "Timed out waiting for builder to register\n"; + + return $self; +} + +sub _spawn { + my ($self, $key, $label, $cmd, %opts) = @_; + my %env = %{ $self->{ctx}{central_env} }; + if ($opts{env}) { + $env{$_} = $opts{env}{$_} for keys %{$opts{env}}; + } + my ($in, $out, $err) = ("", "", ""); + my $harness; + { + local @ENV{keys %env} = values %env; + local $ENV{NO_COLOR} = "1"; + $harness = IPC::Run::start($cmd, \$in, \$out, \$err); + } + $self->{procs}{$key} = { + label => $label, + harness => $harness, + out => \$out, + err => \$err, + }; +} + +sub _spawn_upstream { + my ($self) = @_; + # socat keeps the listener alive across connections by forking a + # fresh `nix-daemon --stdio` per accept. The legacy command honours + # NIX_STORE_DIR / NIX_STATE_DIR from central_env, which already + # points at the test's on-disk store. + $self->_spawn( + upstream => "Upstream nix daemon", + [ + "socat", + "UNIX-LISTEN:$self->{upstream_sock},fork,reuseaddr,unlink-early", + "EXEC:nix-daemon --stdio,nofork", + ], + ); + wait_for_socket($self->{upstream_sock}) + or die "upstream nix-daemon socket did not appear at $self->{upstream_sock}\n"; +} + +sub _spawn_drv_daemon { + my ($self) = @_; + my $ctx = $self->{ctx}; + my $db_url = $ctx->{central}{hydra_database_url}; + my $store_dir = $ctx->{central}{nix_store_dir}; + + $self->_spawn( + drv_daemon => "drv-daemon", + [ + "hydra-drv-daemon", + "--socket", $self->{daemon_sock}, + "--upstream-socket", $self->{upstream_sock}, + "--db-url", $db_url, + "--store-dir", $store_dir, + ], + env => { RUST_LOG => "hydra_drv_daemon=debug,info" }, + ); + wait_for_socket($self->{daemon_sock}) + or die "hydra-drv-daemon socket did not appear at $self->{daemon_sock}\n"; +} + +sub daemon_socket { return $_[0]->{daemon_sock}; } + +# `unix://` ignores NIX_STORE_DIR for its logical store path; the only +# way to make the client agree with the daemon is to pass +# `?store=` as a URL parameter. `/` and `:` stay literal so the +# URL is human-readable; anything else gets percent-encoded. +sub nix_remote_url { + my ($self) = @_; + my $store_dir = $self->{ctx}{central}{nix_store_dir}; + return + "unix://" . uri_escape($self->{daemon_sock}, "^A-Za-z0-9\\-_.~/:") + . "?store=" . uri_escape($store_dir, "^A-Za-z0-9\\-_.~/:"); +} + +sub pump_logs { + my ($self) = @_; + for my $key (keys %{$self->{procs}}) { + my $p = $self->{procs}{$key}; + eval { $p->{harness}->pump_nb }; + _flush_proc($p); + } +} + +# Block until the queue runner reports no in-flight builds for any of +# @build_ids. Bails out if the builder dies. +sub wait_for_builds_to_finish { + my ($self, @build_ids) = @_; + my $ua = $self->{ua}; + my $base_url = $self->{base_url}; + + my $timeout = 60 * scalar(@build_ids); + $timeout = 60 if $timeout < 60; + my $deadline = time() + $timeout; + while (time() < $deadline) { + $self->pump_logs; + my $bl = $self->{procs}{builder}{harness}; + if ($bl && !$bl->pumpable) { + $bl->finish; + my $rc = $bl->result; + die "builder exited unexpectedly (exit code $rc)\n"; + } + + my $all_done = 1; + for my $bid (@build_ids) { + my $resp = $ua->get("$base_url/status/build/$bid/active"); + if ($resp->decoded_content =~ /true/) { + $all_done = 0; + last; + } + } + return 1 if $all_done; + sleep 2; + } + die "timed out waiting for builds to finish\n"; +} + +sub run_cmd { + my ($self, $timeout, @cmd) = @_; + my %env = ( + %{ $self->{ctx}{central_env} }, + NIX_REMOTE => $self->nix_remote_url, + ); + + my ($cmd_in, $cmd_out, $cmd_err) = ("", "", ""); + my $h; + { + local @ENV{keys %env} = values %env; + local $ENV{NO_COLOR} = "1"; + $h = IPC::Run::start(\@cmd, \$cmd_in, \$cmd_out, \$cmd_err); + } + + my $deadline = time() + $timeout; + while (time() < $deadline) { + eval { $h->pump_nb }; + my $err = $@; + # Flush daemon-side logs so yath's event-timeout doesn't trigger + # while we wait for the client to come back. + $self->pump_logs; + if ($err) { + return (1, $cmd_out, $cmd_err . "\n[run_cmd: pump_nb error: $err]"); + } + if (!$h->pumpable) { + $h->finish; + my $rc = scalar $h->result; + unless (defined $rc) { + return ( + 1, $cmd_out, + $cmd_err . "\n[run_cmd: IPC::Run::result returned undef; child likely crashed without a clean exit]", + ); + } + return ($rc, $cmd_out, $cmd_err); + } + select(undef, undef, undef, 0.5); + } + eval { $h->kill_kill }; + return (1, $cmd_out, $cmd_err . "\n[run_cmd: timed out after ${timeout}s]"); +} + +sub stop { + my ($self) = @_; + return if $self->{stopped}; + $self->{stopped} = 1; + for my $key (qw(builder queue_runner drv_daemon upstream)) { + my $p = $self->{procs}{$key}; + next unless $p; + eval { $p->{harness}->kill_kill }; + _flush_proc($p, 1); + } +} + +sub DESTROY { + my ($self) = @_; + $self->stop; +} + +1; diff --git a/subprojects/hydra-tests/lib/HydraTestContext.pm b/subprojects/hydra-tests/lib/HydraTestContext.pm index 58d604b5f..72188dd75 100644 --- a/subprojects/hydra-tests/lib/HydraTestContext.pm +++ b/subprojects/hydra-tests/lib/HydraTestContext.pm @@ -196,6 +196,21 @@ sub jobsdir { return $self->{jobsdir}; } +# Lazily spawn a DrvDaemonContext when HYDRA_TEST_USE_DRV_DAEMON is +# set in the environment; returns undef otherwise. Eval-using helpers +# (CliRunners::evalSucceeds etc.) call this and route their nix-daemon +# traffic through the daemon socket if a daemon is available, so the +# whole eval-using test suite can be exercised both with and without +# the drv-daemon by toggling the env. +sub drv_daemon { + my ($self) = @_; + return $self->{_drv_daemon} if defined $self->{_drv_daemon}; + return undef unless $ENV{HYDRA_TEST_USE_DRV_DAEMON}; + require DrvDaemonContext; + $self->{_drv_daemon} = DrvDaemonContext->new($self); + return $self->{_drv_daemon}; +} + sub nix_state_dir { my ($self) = @_; diff --git a/subprojects/hydra-tests/lib/QueueRunnerContext.pm b/subprojects/hydra-tests/lib/QueueRunnerContext.pm index 78c04fb54..5ed01b461 100644 --- a/subprojects/hydra-tests/lib/QueueRunnerContext.pm +++ b/subprojects/hydra-tests/lib/QueueRunnerContext.pm @@ -10,7 +10,9 @@ use POSIX qw(dup2); use Hydra::Config; our @ISA = qw(Exporter); our @EXPORT = qw( + start_builder start_queue_runner + wait_for_socket wait_for_url ); @@ -26,6 +28,15 @@ sub wait_for_url { return 0; } +sub wait_for_socket { + my ($path) = @_; + for my $i (1..60) { + return 1 if -S $path; + select(undef, undef, undef, 0.5); + } + return 0; +} + # Start a nix daemon for the given store config. # Returns the daemon harness. Caller must kill_kill it when done. sub start_nix_daemon { @@ -60,6 +71,13 @@ sub start_nix_daemon { # pass them to the queue runner via LISTEN_FDS/LISTEN_FDNAMES. # Returns ($harness, $rest_url, $grpc_addr, \$stdout_buf, \$stderr_buf, $daemon_harness). # Caller is responsible for calling $harness->kill_kill when done. +# +# Options: +# rust_log: RUST_LOG value, default "error". +# queue_monitor_loop: 1 to leave the queue-monitor-loop running so +# the queue runner picks up new Builds rows on its own (the +# drv-daemon's ad-hoc flow needs this). Default 0 (disabled), which +# matches QueueRunnerBuildOne's manual /build_one driver. sub start_queue_runner { my ($ctx, %opts) = @_; ref $ctx eq 'HydraTestContext' or die "start_queue_runner requires a HydraTestContext\n"; @@ -119,6 +137,14 @@ sub start_queue_runner { my ($qr_in, $qr_out, $qr_err) = ("", "", ""); + my @args = ( + "hydra-queue-runner", + "--config-path", $config_file, + "--rest-bind", "-", + "--grpc-bind", "-", + ); + push @args, "--disable-queue-monitor-loop" unless $opts{queue_monitor_loop}; + # Start the queue runner, connecting to the nix daemon via unix://. my $qr_harness; { @@ -131,12 +157,7 @@ sub start_queue_runner { # Don't set LISTEN_PID — listenfd skips the PID check when it's unset. delete $ENV{LISTEN_PID}; $qr_harness = IPC::Run::start( - ["hydra-queue-runner", - "--config-path", $config_file, - "--rest-bind", "-", - "--grpc-bind", "-", - "--disable-queue-monitor-loop", - ], + \@args, \$qr_in, \$qr_out, \$qr_err, init => sub { # In the child: place sockets at fd 3 and 4. @@ -159,4 +180,29 @@ sub start_queue_runner { return ($qr_harness, $rest_url, $grpc_addr, \$qr_out, \$qr_err, $daemon_harness); } +# Start a hydra-builder against an already-running queue runner. +# Returns ($harness, \$stdout_buf, \$stderr_buf). +sub start_builder { + my ($ctx, $grpc_port, %opts) = @_; + ref $ctx eq 'HydraTestContext' or die "start_builder requires a HydraTestContext\n"; + + my ($bl_in, $bl_out, $bl_err) = ("", "", ""); + my $bl_harness; + { + local $ENV{NIX_REMOTE} = $ctx->{builder}{nix_store_uri}; + local $ENV{NIX_CONF_DIR} = $ctx->{builder}{nix_conf_dir}; + local $ENV{NIX_STATE_DIR} = $ctx->{builder}{nix_state_dir}; + # TODO: hydra-builder reads NIX_STORE_DIR to report its store + # dir to the queue runner; should use the store URI instead. + local $ENV{NIX_STORE_DIR} = $ctx->{builder}{nix_store_dir}; + local $ENV{RUST_LOG} = $opts{rust_log} // "error"; + local $ENV{NO_COLOR} = "1"; + $bl_harness = IPC::Run::start( + ["hydra-builder", "--gateway-endpoint", "http://[::1]:$grpc_port"], + \$bl_in, \$bl_out, \$bl_err, + ); + } + return ($bl_harness, \$bl_out, \$bl_err); +} + 1; diff --git a/subprojects/hydra-tests/meson.build b/subprojects/hydra-tests/meson.build index deda7f5cd..f9c9cd948 100644 --- a/subprojects/hydra-tests/meson.build +++ b/subprojects/hydra-tests/meson.build @@ -7,12 +7,10 @@ fs = import('fs') perl = find_program('perl', native: true) nix = find_program('nix') -hydra_evaluator = find_program('hydra-evaluator') curl = find_program('curl') exe_path = [ fs.parent(nix.full_path()), - fs.parent(hydra_evaluator.full_path()), fs.parent(curl.full_path()), meson.current_source_dir() / 'scripts', ] @@ -34,6 +32,7 @@ if meson.is_subproject() else hydra_queue_runner = find_program('hydra-queue-runner') hydra_builder = find_program('hydra-builder') + hydra_evaluator = find_program('hydra-evaluator') hydra_init = find_program('hydra-init') hydra_notify = find_program('hydra-notify') @@ -42,6 +41,7 @@ else exe_path += fs.parent(hydra_queue_runner.full_path()) exe_path += fs.parent(hydra_builder.full_path()) + exe_path += fs.parent(hydra_evaluator.full_path()) exe_path += fs.parent(hydra_init.full_path()) exe_path += fs.parent(hydra_notify.full_path()) endif @@ -73,3 +73,10 @@ test('testsuite', workdir: meson.current_source_dir(), timeout: -1, ) + +# A second target that runs the eval-using tests with HYDRA_TEST_USE_DRV_DAEMON=1 +# is intentionally absent until harmonia-protocol supports the +# `fixed:r:sha256` CA hash algorithm nix-eval-jobs sends through the +# daemon during evaluation. The plumbing for it lives in +# HydraTestContext::drv_daemon + CliRunners; once harmonia is bumped +# the testsuite-drv-daemon target can be added back. diff --git a/subprojects/hydra-tests/package.nix b/subprojects/hydra-tests/package.nix index 613cb8d58..5cba9f47c 100644 --- a/subprojects/hydra-tests/package.nix +++ b/subprojects/hydra-tests/package.nix @@ -6,9 +6,12 @@ hydra, hydra-queue-runner, hydra-builder, + hydra-drv-daemon, + hydra-evaluator, meson, ninja, + socat, perl, nixComponents, @@ -59,6 +62,8 @@ stdenv.mkDerivation (finalAttrs: { hydra hydra-queue-runner hydra-builder + hydra-drv-daemon + hydra-evaluator hydra.perlDeps perl nixComponents.nix-cli @@ -72,6 +77,7 @@ stdenv.mkDerivation (finalAttrs: { postgresql_17 pixz nix-eval-jobs + socat curl ]; diff --git a/subprojects/hydra/hydra-evaluator/hydra-evaluator.cc b/subprojects/hydra/hydra-evaluator/hydra-evaluator.cc deleted file mode 100644 index e1d55165a..000000000 --- a/subprojects/hydra/hydra-evaluator/hydra-evaluator.cc +++ /dev/null @@ -1,612 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -using namespace nix; - -struct Connection : pqxx::connection -{ - Connection() : pqxx::connection(getFlags()) { }; - - std::string getFlags() - { - auto s = getEnv("HYDRA_DBI").value_or("dbi:Pg:dbname=hydra;"); - - std::string lower_prefix = "dbi:Pg:"; - std::string upper_prefix = "DBI:Pg:"; - - if (hasPrefix(s, lower_prefix) || hasPrefix(s, upper_prefix)) { - return concatStringsSep(" ", tokenizeString(std::string(s, lower_prefix.size()), ";")); - } - - throw Error("$HYDRA_DBI does not denote a PostgreSQL database"); - } -}; - - -class receiver -{ - std::optional status; - pqxx::connection & conn; - -public: - - receiver(pqxx::connection_base & c, const std::string & channel) - : conn(static_cast(c)) - { - conn.listen(channel, [this](pqxx::notification n) { - status = std::string(n.payload); - }); - } - - std::optional get() { - auto s = status; - status = std::nullopt; - return s; - } -}; - - -struct HydraConfig -{ - std::map options; - - HydraConfig() - { - /* Read hydra.conf. */ - auto hydraConfigFile = getEnv("HYDRA_CONFIG"); - if (hydraConfigFile && pathExists(*hydraConfigFile)) { - - for (auto line : tokenizeString(readFile(*hydraConfigFile), "\n")) { - line = trim(std::string(line, 0, line.find('#'))); - - auto eq = line.find('='); - if (eq == std::string::npos) continue; - - auto key = trim(std::string(line, 0, eq)); - auto value = trim(std::string(line, eq + 1)); - - if (key == "") continue; - - options[key] = value; - } - } - } - - std::string getStrOption(const std::string & key, const std::string & def = "") - { - auto i = options.find(key); - return i == options.end() ? def : i->second; - } - - uint64_t getIntOption(const std::string & key, uint64_t def = 0) - { - auto i = options.find(key); - return i == options.end() ? def : std::stoll(i->second); - } - - bool getBoolOption(const std::string & key, bool def = false) - { - auto i = options.find(key); - return i == options.end() ? def : (i->second == "true" || i->second == "1"); - } -}; - -typedef std::pair JobsetName; - -class JobsetId { - public: - - std::string project; - std::string jobset; - int id; - - - JobsetId(const std::string & project, const std::string & jobset, int id) - : project{ project }, jobset{ jobset }, id{ id } - { - } - - friend bool operator== (const JobsetId & lhs, const JobsetId & rhs); - friend bool operator!= (const JobsetId & lhs, const JobsetId & rhs); - friend bool operator< (const JobsetId & lhs, const JobsetId & rhs); - - - friend bool operator== (const JobsetId & lhs, const JobsetName & rhs); - friend bool operator!= (const JobsetId & lhs, const JobsetName & rhs); - - std::string display() const { - return boost::str(boost::format("%1%:%2% (jobset#%3%)") % project % jobset % id); - } -}; -bool operator==(const JobsetId & lhs, const JobsetId & rhs) -{ - return lhs.id == rhs.id; -} - -bool operator!=(const JobsetId & lhs, const JobsetId & rhs) -{ - return lhs.id != rhs.id; -} - -bool operator<(const JobsetId & lhs, const JobsetId & rhs) -{ - return lhs.id < rhs.id; -} - -bool operator==(const JobsetId & lhs, const JobsetName & rhs) -{ - return lhs.project == rhs.first && lhs.jobset == rhs.second; -} - -bool operator!=(const JobsetId & lhs, const JobsetName & rhs) -{ - return ! (lhs == rhs); -} - -enum class EvaluationStyle -{ - SCHEDULE = 1, - ONESHOT = 2, - ONE_AT_A_TIME = 3, -}; - -struct Evaluator -{ - std::unique_ptr config; - - nix::Pool dbPool; - - struct Jobset - { - JobsetId name; - std::optional evaluation_style; - time_t lastCheckedTime, triggerTime; - int checkInterval; - Pid pid; - }; - - typedef std::map Jobsets; - - std::optional evalOne; - - const size_t maxEvals; - - struct State - { - size_t runningEvals = 0; - Jobsets jobsets; - }; - - Sync state_; - - std::condition_variable childStarted; - std::condition_variable maybeDoWork; - - const time_t notTriggered = std::numeric_limits::max(); - - Evaluator() - : config(std::make_unique()) - , maxEvals(std::max((size_t) 1, (size_t) config->getIntOption("max_concurrent_evals", 4))) - { } - - void readJobsets() - { - auto conn(dbPool.get()); - - pqxx::work txn(*conn); - - auto res = txn.exec - ("select j.id as id, project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled " - "from Jobsets j " - "join Projects p on j.project = p.name " - "where j.enabled != 0 and p.enabled != 0"); - - - auto state(state_.lock()); - - std::set seen; - - for (auto const & row : res) { - auto name = JobsetId{row["project"].as(), row["name"].as(), row["id"].as()}; - - if (evalOne && name != *evalOne) continue; - - auto res = state->jobsets.try_emplace(name, Jobset{name}); - - auto & jobset = res.first->second; - jobset.lastCheckedTime = row["lastCheckedTime"].as(0); - jobset.triggerTime = row["triggerTime"].as(notTriggered); - jobset.checkInterval = row["checkInterval"].as(); - switch (row["jobset_enabled"].as(0)) { - case 1: - jobset.evaluation_style = EvaluationStyle::SCHEDULE; - break; - case 2: - jobset.evaluation_style = EvaluationStyle::ONESHOT; - break; - case 3: - jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME; - break; - } - - seen.insert(name); - } - - if (evalOne && seen.empty()) { - printError("the specified jobset does not exist or is disabled"); - std::_Exit(1); - } - - for (auto i = state->jobsets.begin(); i != state->jobsets.end(); ) - if (seen.count(i->first)) - ++i; - else { - printInfo("forgetting jobset ‘%s’", i->first.display()); - i = state->jobsets.erase(i); - } - } - - void startEval(State & state, Jobset & jobset) - { - time_t now = time(0); - - printInfo("starting evaluation of jobset ‘%s’ (last checked %d s ago)", - jobset.name.display(), - now - jobset.lastCheckedTime); - - { - auto conn(dbPool.get()); - pqxx::work txn(*conn); - txn.exec("update Jobsets set startTime = $1 where id = $2", - pqxx::params{now, jobset.name.id}).no_rows(); - txn.commit(); - } - - assert(jobset.pid == -1); - - jobset.pid = startProcess([&]() { - Strings args = { "hydra-eval-jobset", jobset.name.project, jobset.name.jobset }; - execvp(args.front().c_str(), stringsToCharPtrs(args).data()); - throw SysError("executing ‘%1%’", args.front()); - }); - - state.runningEvals++; - - childStarted.notify_one(); - } - - bool shouldEvaluate(Jobset & jobset) - { - if (jobset.pid != -1) { - // Already running. - debug("shouldEvaluate %s? no: already running", - jobset.name.display()); - return false; - } - - if (jobset.triggerTime != std::numeric_limits::max()) { - // An evaluation of this Jobset is requested - debug("shouldEvaluate %s? yes: requested", - jobset.name.display()); - return true; - } - - if (jobset.checkInterval <= 0) { - // Automatic scheduling is disabled. We allow requested - // evaluations, but never schedule start one. - debug("shouldEvaluate %s? no: checkInterval <= 0", - jobset.name.display()); - return false; - } - - if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) { - // Time to schedule a fresh evaluation. If the jobset - // is a ONE_AT_A_TIME jobset, ensure the previous jobset - // has no remaining, unfinished work. - - auto conn(dbPool.get()); - - pqxx::work txn(*conn); - - if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) { - auto evaluation_res = txn.exec - ("select id from JobsetEvals " - "where jobset_id = $1 " - "order by id desc limit 1" - ,jobset.name.id - ); - - if (evaluation_res.empty()) { - // First evaluation, so allow scheduling. - debug("shouldEvaluate(one-at-a-time) %s? yes: no prior eval", - jobset.name.display()); - return true; - } - - auto evaluation_id = evaluation_res[0][0].as(); - - auto unfinished_build_res = txn.exec - ("select id from Builds " - "join JobsetEvalMembers " - " on (JobsetEvalMembers.build = Builds.id) " - "where JobsetEvalMembers.eval = $1 " - " and builds.finished = 0 " - " limit 1" - ,evaluation_id - ); - - // If the previous evaluation has no unfinished builds - // schedule! - if (unfinished_build_res.empty()) { - debug("shouldEvaluate(one-at-a-time) %s? yes: no unfinished builds", - jobset.name.display()); - return true; - } else { - debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build", - jobset.name.display()); - return false; - } - - - } else { - // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED - debug("shouldEvaluate(oneshot/scheduled) %s? yes: checkInterval elapsed", - jobset.name.display()); - return true; - } - } - - return false; - } - - void startEvals(State & state) - { - std::vector sorted; - - /* Filter out jobsets that have been evaluated recently and have - not been triggered. */ - for (auto i = state.jobsets.begin(); i != state.jobsets.end(); ++i) - if (evalOne || - (i->second.evaluation_style && shouldEvaluate(i->second))) - sorted.push_back(i); - - /* Put jobsets in order of ascending trigger time, last checked - time, and name. */ - std::sort(sorted.begin(), sorted.end(), - [](const Jobsets::iterator & a, const Jobsets::iterator & b) { - return - a->second.triggerTime != b->second.triggerTime - ? a->second.triggerTime < b->second.triggerTime - : a->second.lastCheckedTime != b->second.lastCheckedTime - ? a->second.lastCheckedTime < b->second.lastCheckedTime - : a->first < b->first; - }); - - /* Start jobset evaluations up to the concurrency limit.*/ - for (auto & i : sorted) { - if (state.runningEvals >= maxEvals) break; - startEval(state, i->second); - } - } - - void loop() - { - auto state(state_.lock()); - - while (true) { - - time_t now = time(0); - - std::chrono::seconds sleepTime = std::chrono::seconds::max(); - - if (state->runningEvals < maxEvals) { - for (auto & i : state->jobsets) - if (i.second.pid == -1 && - i.second.checkInterval > 0) - sleepTime = std::min(sleepTime, std::chrono::seconds( - std::max((time_t) 1, i.second.lastCheckedTime - now + i.second.checkInterval))); - } - - debug("waiting for %d s", sleepTime.count()); - if (sleepTime == std::chrono::seconds::max()) - state.wait(maybeDoWork); - else - state.wait_for(maybeDoWork, sleepTime); - - startEvals(*state); - } - } - - /* A thread that listens to PostgreSQL notifications about jobset - changes, updates the jobsets map, and signals the main thread - to start evaluations. */ - void databaseMonitor() - { - while (true) { - - try { - - auto conn(dbPool.get()); - - receiver jobsetsAdded(*conn, "jobsets_added"); - receiver jobsetsDeleted(*conn, "jobsets_deleted"); - receiver jobsetsChanged(*conn, "jobset_scheduling_changed"); - - while (true) { - /* Note: we read/notify before - await_notification() to ensure we don't miss a - state change. */ - readJobsets(); - maybeDoWork.notify_one(); - conn->await_notification(); - printInfo("received jobset event"); - } - - } catch (pqxx::broken_connection & e) { - printError("Database connection broken: %s", e.what()); - std::_Exit(1); - } catch (std::exception & e) { - printError("exception in database monitor thread: %s", e.what()); - sleep(30); - } - } - } - - /* A thread that reaps child processes.*/ - void reaper() - { - while (true) { - { - auto state(state_.lock()); - while (!state->runningEvals) - state.wait(childStarted); - } - - int status; - pid_t pid = waitpid(-1, &status, 0); - if (pid == -1) { - if (errno == EINTR) continue; - throw SysError("waiting for children"); - } - - { - auto state(state_.lock()); - assert(state->runningEvals); - state->runningEvals--; - - // FIXME: should use a map. - for (auto & i : state->jobsets) { - auto & jobset(i.second); - - if (jobset.pid == pid) { - printInfo("evaluation of jobset ‘%s’ %s", - jobset.name.display(), statusToString(status)); - - auto now = time(0); - - jobset.triggerTime = notTriggered; - jobset.lastCheckedTime = now; - - try { - - auto conn(dbPool.get()); - pqxx::work txn(*conn); - - /* Clear the trigger time to prevent this - jobset from getting stuck in an endless - failing eval loop. */ - txn.exec - ("update Jobsets set triggerTime = null where id = $1 and startTime is not null and triggerTime <= startTime", - jobset.name.id).no_rows(); - - /* Clear the start time. */ - txn.exec - ("update Jobsets set startTime = null where id = $1", - jobset.name.id).no_rows(); - - if (!WIFEXITED(status) || WEXITSTATUS(status) > 1) { - txn.exec("update Jobsets set errorMsg = $1, lastCheckedTime = $2, errorTime = $2, fetchErrorMsg = null where id = $3", - pqxx::params{fmt("evaluation %s", statusToString(status)), now, jobset.name.id}).no_rows(); - } - - txn.commit(); - - } catch (std::exception & e) { - printError("exception setting jobset error: %s", e.what()); - } - - jobset.pid.release(); - maybeDoWork.notify_one(); - - if (evalOne) std::_Exit(0); - - break; - } - } - } - } - } - - void unlock() - { - auto conn(dbPool.get()); - pqxx::work txn(*conn); - txn.exec("update Jobsets set startTime = null").no_rows(); - txn.commit(); - } - - void run() - { - unlock(); - - /* Can't be bothered to shut down cleanly. Goodbye! */ - auto callback = createInterruptCallback([&]() { std::_Exit(1); }); - - std::thread reaperThread([&]() { reaper(); }); - - std::thread monitorThread([&]() { databaseMonitor(); }); - - while (true) { - try { - loop(); - } catch (pqxx::broken_connection & e) { - printError("Database connection broken: %s", e.what()); - std::_Exit(1); - } catch (std::exception & e) { - printError("exception in main loop: %s", e.what()); - sleep(30); - } - } - } -}; - -int main(int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - - signal(SIGINT, SIG_DFL); - signal(SIGTERM, SIG_DFL); - signal(SIGHUP, SIG_DFL); - - bool unlock = false; - - Evaluator evaluator; - - std::vector args; - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--unlock") - unlock = true; - else if (hasPrefix(*arg, "-")) - return false; - args.push_back(*arg); - return true; - }); - - - if (unlock) - evaluator.unlock(); - else { - if (!args.empty()) { - if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [ ]"); - evaluator.evalOne = JobsetName(args[0], args[1]); - } - evaluator.run(); - } - }); -} diff --git a/subprojects/hydra/hydra-evaluator/meson.build b/subprojects/hydra/hydra-evaluator/meson.build deleted file mode 100644 index 76561071c..000000000 --- a/subprojects/hydra/hydra-evaluator/meson.build +++ /dev/null @@ -1,9 +0,0 @@ -hydra_evaluator = executable('hydra-evaluator', - 'hydra-evaluator.cc', - dependencies: [ - nix_util_dep, - nix_main_dep, - pqxx_dep, - ], - install: true, -) diff --git a/subprojects/hydra/meson.build b/subprojects/hydra/meson.build index 6a0f46a87..c6bb71085 100644 --- a/subprojects/hydra/meson.build +++ b/subprojects/hydra/meson.build @@ -1,28 +1,12 @@ -project('hydra', 'cpp', +project('hydra', version: files('version.txt'), license: 'GPL-3.0', default_options: [ 'debug=true', 'optimization=2', - 'cpp_std=c++23', ], ) -nix_util_dep = dependency('nix-util', required: true) -nix_store_dep = dependency('nix-store', required: true) -nix_main_dep = dependency('nix-main', required: true) - -pqxx_dep = dependency('libpqxx', required: true) - -prom_cpp_core_dep = dependency('prometheus-cpp-core', required: true) -prom_cpp_pull_dep = dependency('prometheus-cpp-pull', required: true) - -# Native code -subdir('hydra-evaluator') - -# Make executables available to sibling subprojects (e.g. hydra-tests) -meson.override_find_program('hydra-evaluator', hydra_evaluator) - hydra_libexecdir = get_option('libexecdir') / 'hydra' # Data and interpreted diff --git a/subprojects/hydra/package.nix b/subprojects/hydra/package.nix index c72a86fbe..d61bb5366 100644 --- a/subprojects/hydra/package.nix +++ b/subprojects/hydra/package.nix @@ -18,18 +18,11 @@ meson, ninja, nukeReferences, - pkg-config, unzip, - libpqxx, - openssl, bzip2, - libxslt, perl, pixz, - boost, - nlohmann_json, - prometheus-cpp, openssh, coreutils, @@ -138,7 +131,6 @@ stdenv.mkDerivation (finalAttrs: { src = lib.fileset.toSource { root = ../..; fileset = lib.fileset.unions [ - ../../subprojects/hydra/hydra-evaluator ../../subprojects/hydra/lib ../../subprojects/hydra/Makefile.PL ../../subprojects/hydra/meson.build @@ -168,24 +160,14 @@ stdenv.mkDerivation (finalAttrs: { meson ninja nukeReferences - pkg-config perlDeps perl unzip ]; buildInputs = [ - libpqxx - openssl - libxslt - nixComponents.nix-util - nixComponents.nix-store - nixComponents.nix-main perlDeps perl - boost - nlohmann_json - prometheus-cpp ]; hydraPath = lib.makeBinPath (