diff --git a/.gitignore b/.gitignore index 90a2497b3..7dbbbfe4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Editor backup files *~ +# Claude Code local settings +.claude + # Nix dev shell /.direnv/ diff --git a/.sqlx/query-7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5.json b/.sqlx/query-4ecbc901d73fa59738d8826750482942962686992e1d16256771d1bb4af72f4a.json similarity index 59% rename from .sqlx/query-7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5.json rename to .sqlx/query-4ecbc901d73fa59738d8826750482942962686992e1d16256771d1bb4af72f4a.json index a04662e97..20fd11d12 100644 --- a/.sqlx/query-7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5.json +++ b/.sqlx/query-4ecbc901d73fa59738d8826750482942962686992e1d16256771d1bb4af72f4a.json @@ -1,40 +1,50 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n type,\n subtype,\n fileSize,\n sha256hash,\n path,\n name,\n defaultPath\n FROM buildproducts\n WHERE build = $1 ORDER BY productnr;", + "query": "\n SELECT\n build,\n productnr,\n type,\n subtype,\n fileSize,\n sha256hash,\n path,\n name,\n defaultPath\n FROM buildproducts\n WHERE build = $1 ORDER BY productnr;", "describe": { "columns": [ { "ordinal": 0, + "name": "build", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "productnr", + "type_info": "Int4" + }, + { + "ordinal": 2, "name": "type", "type_info": "Text" }, { - "ordinal": 1, + "ordinal": 3, "name": "subtype", "type_info": "Text" }, { - "ordinal": 2, + "ordinal": 4, "name": "filesize", "type_info": "Int8" }, { - "ordinal": 3, + "ordinal": 5, "name": "sha256hash", "type_info": "Text" }, { - "ordinal": 4, + "ordinal": 6, "name": "path", "type_info": "Text" }, { - "ordinal": 5, + "ordinal": 7, "name": "name", "type_info": "Text" }, { - "ordinal": 6, + "ordinal": 8, "name": "defaultpath", "type_info": "Text" } @@ -45,6 +55,8 @@ ] }, "nullable": [ + false, + false, false, false, true, @@ -54,5 +66,5 @@ true ] }, - "hash": "7b0ede437aa972b2588415a5bfe8ff545285d457dc809601c9c780c955e16ff5" + "hash": "4ecbc901d73fa59738d8826750482942962686992e1d16256771d1bb4af72f4a" } diff --git a/.sqlx/query-694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006.json b/.sqlx/query-694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006.json deleted file mode 100644 index 1f2bcfa96..000000000 --- a/.sqlx/query-694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO systemstatus (\n what, status\n ) VALUES (\n 'queue-runner', $1\n ) ON CONFLICT (what) DO UPDATE SET status = EXCLUDED.status;", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Json" - ] - }, - "nullable": [] - }, - "hash": "694cba4e8063ef370358e1c5828b9450c3c06b8449eeee76f2a3a5131a6e3006" -} diff --git a/.sqlx/query-af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7.json b/.sqlx/query-af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7.json deleted file mode 100644 index b7363a23b..000000000 --- a/.sqlx/query-af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status FROM systemstatus WHERE what = 'queue-runner';", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "status", - "type_info": "Json" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "af02dbe27d340f0ee8ed4803d3a1ecf1fa7268064dba9130bbd833f1f484e2a7" -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..402e765c9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ +# Contributing to Hydra + +## Formatting Markdown + +We use [semantic line breaks](https://sembr.org/) in all Markdown files. +This means each sentence (or independent clause) should start on its own line. + +Semantic line breaks make diffs easier to read and review, since a change to one sentence won't reflow an entire paragraph. +They also make it easier to rearrange and edit prose. diff --git a/Cargo.lock b/Cargo.lock index 2dd13f9df..d4e9e9187 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -116,13 +125,19 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" @@ -131,9 +146,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-compression" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -214,9 +229,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "axum" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", @@ -266,6 +281,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.22.1" @@ -286,15 +316,24 @@ dependencies = [ "backon", "bytes", "configparser", + "daemon-client-utils", "foldhash 0.2.0", "fs-err", "futures", - "harmonia-store-core", + "harmonia-file-core", + "harmonia-file-nar", + "harmonia-protocol", + "harmonia-store-derivation", + "harmonia-store-nar-info", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-store-remote", "harmonia-utils-hash", + "harmonia-utils-io", + "harmonia-utils-signature", "hashbrown 0.16.1", "hydra-tracing", "moka", - "nix-utils", "object_store", "reqwest", "secrecy", @@ -313,9 +352,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" dependencies = [ "serde_core", ] @@ -332,6 +371,20 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -395,6 +448,17 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -461,9 +525,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.58" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -491,7 +555,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -508,9 +572,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -530,9 +594,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -547,14 +611,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] -name = "codespan-reporting" -version = "0.13.1" +name = "color-eyre" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d" dependencies = [ - "serde", - "termcolor", - "unicode-width", + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", ] [[package]] @@ -565,9 +645,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "compression-codecs" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "brotli", "bzip2", @@ -579,9 +659,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -610,6 +690,12 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "convert_case" version = "0.10.0" @@ -674,9 +760,9 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" @@ -767,65 +853,16 @@ dependencies = [ ] [[package]] -name = "cxx" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" -dependencies = [ - "cc", - "cxx-build", - "cxxbridge-cmd", - "cxxbridge-flags", - "cxxbridge-macro", - "foldhash 0.2.0", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" -dependencies = [ - "cc", - "codespan-reporting", - "indexmap", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.117", -] - -[[package]] -name = "cxxbridge-cmd" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" -dependencies = [ - "clap", - "codespan-reporting", - "indexmap", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" +name = "daemon-client-utils" +version = "0.1.0" dependencies = [ - "indexmap", - "proc-macro2", - "quote", - "syn 2.0.117", + "harmonia-protocol", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-store-remote", + "petgraph", + "url", ] [[package]] @@ -838,14 +875,18 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" name = "db" version = "0.1.0" dependencies = [ - "anyhow", "futures", - "harmonia-store-core", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-utils-hash", "hashbrown 0.16.1", "jiff", + "nix-support", "serde_json", "sqlx", + "store-path-utils", "test-utils", + "thiserror", "tokio", "tracing", ] @@ -1020,11 +1061,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fiat-crypto" @@ -1263,11 +1314,17 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "gloo-timers" version = "0.3.0" @@ -1282,9 +1339,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1299,12 +1356,94 @@ dependencies = [ "tracing", ] +[[package]] +name = "harmonia-file-core" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "harmonia-file-nar" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "bstr", + "bytes", + "derive_more", + "futures-core", + "futures-sink", + "futures-util", + "harmonia-file-core", + "harmonia-utils-io", + "nix", + "pin-project-lite", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "walkdir", +] + +[[package]] +name = "harmonia-protocol" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "async-stream", + "bstr", + "bytes", + "derive_more", + "futures-core", + "futures-util", + "harmonia-file-nar", + "harmonia-protocol-derive", + "harmonia-store-aterm", + "harmonia-store-build-result", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "harmonia-utils-io", + "harmonia-utils-signature", + "libc", + "num_enum", + "pin-project-lite", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "harmonia-protocol-derive" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "harmonia-store-aterm" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#d1fd4bcc8d87bd5740dc3bd861a9bad343eef9a5" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" dependencies = [ - "harmonia-store-core", + "bytes", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-utils-base-encoding", "harmonia-utils-hash", "memchr", "serde_json", @@ -1312,30 +1451,110 @@ dependencies = [ ] [[package]] -name = "harmonia-store-core" +name = "harmonia-store-build-result" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "harmonia-store-derivation", + "num_enum", + "serde", + "serde_json", +] + +[[package]] +name = "harmonia-store-content-address" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "derive_more", + "harmonia-store-path", + "harmonia-utils-hash", + "serde", + "thiserror", +] + +[[package]] +name = "harmonia-store-derivation" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#d1fd4bcc8d87bd5740dc3bd861a9bad343eef9a5" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" dependencies = [ "bytes", "data-encoding", "derive_more", - "ed25519-dalek", - "getrandom 0.4.2", + "harmonia-store-content-address", + "harmonia-store-path", "harmonia-utils-base-encoding", "harmonia-utils-hash", + "harmonia-utils-signature", "serde", "serde_json", - "subtle", "thiserror", - "tracing", "zerocopy", - "zeroize", +] + +[[package]] +name = "harmonia-store-nar-info" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", + "thiserror", +] + +[[package]] +name = "harmonia-store-path" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "derive_more", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "serde", + "thiserror", + "zerocopy", +] + +[[package]] +name = "harmonia-store-path-info" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", +] + +[[package]] +name = "harmonia-store-remote" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "async-stream", + "futures-core", + "futures-util", + "harmonia-file-nar", + "harmonia-protocol", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-utils-io", + "harmonia-utils-signature", + "prometheus", + "tokio", + "tracing", ] [[package]] name = "harmonia-utils-base-encoding" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#d1fd4bcc8d87bd5740dc3bd861a9bad343eef9a5" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" dependencies = [ "data-encoding", "derive_more", @@ -1345,12 +1564,14 @@ dependencies = [ [[package]] name = "harmonia-utils-hash" version = "0.0.0-alpha.0" -source = "git+https://github.com/nix-community/harmonia.git#d1fd4bcc8d87bd5740dc3bd861a9bad343eef9a5" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" dependencies = [ + "blake3", "data-encoding", "derive_more", "harmonia-utils-base-encoding", "md5", + "pin-project-lite", "serde", "sha1 0.11.0", "sha2 0.11.0", @@ -1358,6 +1579,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "harmonia-utils-io" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "bytes", + "futures-util", + "libc", + "nix", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "harmonia-utils-signature" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#fec973ce0cfb0d9575bd7a466bdac80acccf66b2" +dependencies = [ + "data-encoding", + "ed25519-dalek", + "getrandom 0.4.2", + "harmonia-utils-base-encoding", + "serde", + "subtle", + "thiserror", + "zeroize", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1391,6 +1640,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.10.0" @@ -1498,9 +1753,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ "typenum", ] @@ -1509,30 +1764,38 @@ dependencies = [ name = "hydra-builder" version = "0.1.0" dependencies = [ - "anyhow", "async-compression", "async-stream", "backon", "binary-cache", "bytes", "clap", + "color-eyre", + "daemon-client-utils", "fs-err", "futures", "gethostname", - "harmonia-store-core", + "harmonia-protocol", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-store-remote", + "harmonia-utils-hash", "hashbrown 0.16.1", + "http", + "hydra-proto", "hydra-tracing", "hyper-util", "nix", - "nix-utils", + "nix-support", "parking_lot", "procfs-core 0.18.0", "prost", "sd-notify", "serde", "serde_json", - "sha2 0.10.9", - "shared", + "store-path-utils", + "store-transfer", "sysinfo", "thiserror", "tikv-jemallocator", @@ -1541,18 +1804,41 @@ dependencies = [ "tokio-util", "tonic", "tonic-prost", - "tonic-prost-build", "tower", "tracing", "url", "uuid", ] +[[package]] +name = "hydra-proto" +version = "0.1.0" +dependencies = [ + "bytes", + "db", + "fs-err", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-nar-info", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "nix-support", + "prost", + "serde_json", + "sha2 0.10.9", + "store-path-utils", + "thiserror", + "tonic", + "tonic-prost", + "tonic-prost-build", +] + [[package]] name = "hydra-queue-runner" version = "0.1.0" dependencies = [ - "anyhow", "arc-swap", "async-compression", "atomic_float", @@ -1561,20 +1847,31 @@ dependencies = [ "byte-unit", "bytes", "clap", + "color-eyre", + "daemon-client-utils", "db", "fs-err", "futures", "futures-util", "h2", - "harmonia-store-core", + "harmonia-protocol", + "harmonia-store-aterm", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-nar-info", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-store-remote", + "harmonia-utils-hash", "hashbrown 0.16.1", "http-body-util", + "hydra-proto", "hydra-tracing", "hyper", "hyper-util", "jiff", "listenfd", - "nix-utils", + "nix-support", "parking_lot", "procfs 0.18.0", "prometheus", @@ -1583,9 +1880,8 @@ dependencies = [ "secrecy", "serde", "serde_json", - "sha2 0.10.9", - "shared", "smallvec", + "store-transfer", "thiserror", "tikv-jemallocator", "tokio", @@ -1595,7 +1891,6 @@ dependencies = [ "tonic", "tonic-health", "tonic-prost", - "tonic-prost-build", "tonic-reflection", "tower", "tower-http", @@ -1607,15 +1902,17 @@ dependencies = [ name = "hydra-tracing" version = "0.1.0" dependencies = [ - "anyhow", + "color-eyre", "http", "opentelemetry", "opentelemetry-http", "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", + "thiserror", "tonic", "tracing", + "tracing-error", "tracing-log", "tracing-opentelemetry", "tracing-subscriber", @@ -1623,9 +1920,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -1638,7 +1935,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1646,20 +1942,19 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http", "hyper", "hyper-util", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] @@ -1726,12 +2021,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1739,9 +2035,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1752,9 +2048,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1766,15 +2062,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1786,15 +2082,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1824,22 +2120,28 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", ] +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1850,16 +2152,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1883,9 +2175,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1898,9 +2190,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -1934,9 +2226,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.92" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", @@ -1961,15 +2253,15 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libbz2-rs-sys" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" +checksum = "f8fc329e1457d97a9d58a4e2ca49e3be572431a7e096008efc2e3a3c19d428f4" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "liblzma" @@ -1983,9 +2275,9 @@ dependencies = [ [[package]] name = "liblzma-sys" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" +checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" dependencies = [ "cc", "libc", @@ -2000,14 +2292,14 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ "bitflags", "libc", "plain", - "redox_syscall 0.7.3", + "redox_syscall 0.7.5", ] [[package]] @@ -2020,15 +2312,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "link-cplusplus" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2054,9 +2337,9 @@ dependencies = [ [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2116,6 +2399,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2171,40 +2463,30 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "nix" -version = "0.31.2" +version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ "bitflags", "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] -name = "nix-utils" +name = "nix-support" version = "0.1.0" dependencies = [ - "anyhow", - "bytes", - "cxx", - "cxx-build", "fs-err", - "futures", - "harmonia-store-aterm", - "harmonia-store-core", + "harmonia-store-derivation", + "harmonia-store-path", "harmonia-utils-hash", - "hashbrown 0.16.1", - "pkg-config", - "serde", - "serde_json", - "smallvec", - "thiserror", + "regex", + "sha2 0.10.9", + "store-path-utils", "tokio", - "tokio-stream", - "tokio-util", "tracing", - "url", ] [[package]] @@ -2236,7 +2518,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -2281,6 +2563,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -2300,6 +2604,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" version = "0.13.2" @@ -2323,7 +2636,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.10.0", + "rand 0.10.1", "reqwest", "ring", "serde", @@ -2432,12 +2745,18 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "thiserror", "tokio", "tokio-stream", ] +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + [[package]] name = "parking" version = "2.2.1" @@ -2495,18 +2814,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" dependencies = [ "proc-macro2", "quote", @@ -2519,12 +2838,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkcs1" version = "0.7.5" @@ -2548,9 +2861,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plain" @@ -2566,18 +2879,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2776,9 +3089,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.2" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ "memchr", "serde", @@ -2813,7 +3126,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -2868,9 +3181,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2879,9 +3192,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2889,13 +3202,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.2", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -2938,9 +3251,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "redox_syscall" @@ -2953,9 +3266,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" dependencies = [ "bitflags", ] @@ -3061,7 +3374,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] @@ -3129,21 +3442,27 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.41.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +checksum = "0c5108e3d4d903e21aac27f12ba5377b6b34f9f44b325e4894c7924169d06995" dependencies = [ "arrayvec", "borsh", "bytes", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "rkyv", "serde", "serde_json", "wasm-bindgen", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -3187,9 +3506,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -3214,9 +3533,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -3281,12 +3600,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scratch" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" - [[package]] name = "sd-notify" version = "0.5.0" @@ -3337,9 +3650,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -3386,9 +3699,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] @@ -3458,20 +3771,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shared" -version = "0.1.0" -dependencies = [ - "anyhow", - "fs-err", - "nix-utils", - "prost", - "regex", - "sha2 0.10.9", - "tokio", - "tracing", -] - [[package]] name = "shlex" version = "1.3.0" @@ -3670,7 +3969,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.6", "rsa", "serde", "sha1 0.10.6", @@ -3708,7 +4007,7 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", "sha2 0.10.9", @@ -3750,6 +4049,31 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "store-path-utils" +version = "0.1.0" +dependencies = [ + "harmonia-store-path", +] + +[[package]] +name = "store-transfer" +version = "0.1.0" +dependencies = [ + "async-compression", + "futures", + "harmonia-protocol", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-store-remote", + "hashbrown 0.16.1", + "hydra-proto", + "thiserror", + "tokio", + "tonic", + "tracing", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -3869,21 +4193,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "test-utils" version = "0.1.0" @@ -3943,9 +4258,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -3968,9 +4283,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3985,9 +4300,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -4031,9 +4346,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.0+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8195ca05e4eb728f4ba94f3e3291661320af739c4e43779cbdfae82ab239fcc" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ "indexmap", "serde_core", @@ -4046,18 +4361,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.8+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", "toml_datetime", @@ -4067,24 +4382,24 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.0+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.1.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", @@ -4108,14 +4423,14 @@ dependencies = [ "tower-layer", "tower-service", "tracing", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] name = "tonic-build" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ "prettyplease", "proc-macro2", @@ -4125,9 +4440,9 @@ dependencies = [ [[package]] name = "tonic-health" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ff0636fef47afb3ec02818f5bceb4377b8abb9d6a386aeade18bd6212f8eb7" +checksum = "fcfab99db777fba2802f0dfa861d1628d1ae916fb199d29819941f139ae85082" dependencies = [ "prost", "tokio", @@ -4138,9 +4453,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", @@ -4149,9 +4464,9 @@ dependencies = [ [[package]] name = "tonic-prost-build" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ "prettyplease", "proc-macro2", @@ -4165,9 +4480,9 @@ dependencies = [ [[package]] name = "tonic-reflection" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf0685a51e6d02b502ba0764002e766b7f3042aed13d9234925b6ffbfa3fca7" +checksum = "acccd136a4bf19810a1fde9c74edc6129b42a66b44d0c1c8aaa67aeb49a146a7" dependencies = [ "prost", "prost-types", @@ -4198,21 +4513,21 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" dependencies = [ "bitflags", "bytes", "futures-util", "http", "http-body", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -4260,6 +4575,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -4313,9 +4638,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicase" @@ -4356,12 +4681,6 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - [[package]] name = "unicode-xid" version = "0.2.6" @@ -4406,9 +4725,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -4461,11 +4780,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -4474,7 +4793,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] @@ -4485,9 +4804,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.115" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -4498,9 +4817,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.65" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -4508,9 +4827,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.115" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4518,9 +4837,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.115" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -4531,9 +4850,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.115" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -4587,9 +4906,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.92" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -4611,14 +4930,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] @@ -4655,7 +4974,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -4935,9 +5254,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" dependencies = [ "memchr", ] @@ -4951,6 +5270,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -5032,9 +5357,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" @@ -5047,9 +5372,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -5058,9 +5383,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -5090,18 +5415,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -5117,9 +5442,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -5128,9 +5453,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -5139,9 +5464,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 57cbacf9a..6cbeaf9fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ rust-version = "1.91.0" version = "0.1.0" [workspace.dependencies] -anyhow = "1.0.98" arc-swap = "1.7" async-compression = { version = "0.4", default-features = false } async-stream = "0.3" @@ -17,6 +16,7 @@ backon = "1.5.2" byte-unit = "5.1.6" bytes = "1" clap = "4" +color-eyre = "0.6" configparser = "3.1" cxx = "1" cxx-build = "1" @@ -29,10 +29,20 @@ futures = "0.3" futures-util = "0.3" gethostname = "1" h2 = "0.4" +harmonia-file-core = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-file-nar = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-protocol = { git = "https://github.com/nix-community/harmonia.git" } harmonia-store-aterm = { git = "https://github.com/nix-community/harmonia.git" } -harmonia-store-core = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-content-address = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-derivation = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-nar-info = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-path = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-path-info = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-remote = { git = "https://github.com/nix-community/harmonia.git" } harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia.git" } harmonia-utils-hash = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-io = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-signature = { git = "https://github.com/nix-community/harmonia.git" } hashbrown = "0.16" http = "1.1" http-body-util = "0.1" @@ -51,6 +61,7 @@ opentelemetry-otlp = "0.31.0" opentelemetry-semantic-conventions = "0.31.0" opentelemetry_sdk = "0.31.0" parking_lot = "0.12.4" +petgraph = { version = "0.8", default-features = false, features = [ "graphmap", "std" ] } pkg-config = "0.3" procfs = "0.18" procfs-core = "0.18" @@ -86,15 +97,36 @@ tonic-reflection = "0.14" tower = "0.5" tower-http = "0.6" tracing = "0.1" +tracing-error = "0.2" tracing-log = "0.2.0" tracing-opentelemetry = "0.32.0" tracing-subscriber = "0.3.18" url = "2.5.4" uuid = "1.16" zerocopy = { version = "0.8", features = [ "derive" ] } +zstd = "0.13" + +# Local workspace crates +binary-cache = { path = "subprojects/crates/binary-cache" } +daemon-client-utils = { path = "subprojects/crates/daemon-client-utils" } +db = { path = "subprojects/crates/db" } +hydra-proto = { path = "subprojects/crates/proto" } +hydra-tracing = { path = "subprojects/crates/tracing" } +nix-support = { path = "subprojects/crates/nix-support" } +store-path-utils = { path = "subprojects/crates/store-path-utils" } +store-transfer = { path = "subprojects/crates/store-transfer" } +test-utils = { path = "subprojects/crates/test-utils" } [patch.crates-io] -harmonia-store-aterm = { git = "https://github.com/nix-community/harmonia.git" } -harmonia-store-core = { git = "https://github.com/nix-community/harmonia.git" } -harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia.git" } -harmonia-utils-hash = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-file-core = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-file-nar = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-aterm = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-content-address = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-derivation = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-nar-info = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-path = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-store-path-info = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-hash = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-io = { git = "https://github.com/nix-community/harmonia.git" } +harmonia-utils-signature = { git = "https://github.com/nix-community/harmonia.git" } diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 7d1c323be..000000000 --- a/INSTALL +++ /dev/null @@ -1,365 +0,0 @@ -Installation Instructions -************************* - -Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005, -2006, 2007, 2008, 2009 Free Software Foundation, Inc. - - Copying and distribution of this file, with or without modification, -are permitted in any medium without royalty provided the copyright -notice and this notice are preserved. This file is offered as-is, -without warranty of any kind. - -Basic Installation -================== - - Briefly, the shell commands `./configure; make; make install' should -configure, build, and install this package. The following -more-detailed instructions are generic; see the `README' file for -instructions specific to this package. Some packages provide this -`INSTALL' file but do not implement all of the features documented -below. The lack of an optional feature in a given package is not -necessarily a bug. More recommendations for GNU packages can be found -in *note Makefile Conventions: (standards)Makefile Conventions. - - The `configure' shell script attempts to guess correct values for -various system-dependent variables used during compilation. It uses -those values to create a `Makefile' in each directory of the package. -It may also create one or more `.h' files containing system-dependent -definitions. Finally, it creates a shell script `config.status' that -you can run in the future to recreate the current configuration, and a -file `config.log' containing compiler output (useful mainly for -debugging `configure'). - - It can also use an optional file (typically called `config.cache' -and enabled with `--cache-file=config.cache' or simply `-C') that saves -the results of its tests to speed up reconfiguring. Caching is -disabled by default to prevent problems with accidental use of stale -cache files. - - If you need to do unusual things to compile the package, please try -to figure out how `configure' could check whether to do them, and mail -diffs or instructions to the address given in the `README' so they can -be considered for the next release. If you are using the cache, and at -some point `config.cache' contains results you don't want to keep, you -may remove or edit it. - - The file `configure.ac' (or `configure.in') is used to create -`configure' by a program called `autoconf'. You need `configure.ac' if -you want to change it or regenerate `configure' using a newer version -of `autoconf'. - - The simplest way to compile this package is: - - 1. `cd' to the directory containing the package's source code and type - `./configure' to configure the package for your system. - - Running `configure' might take a while. While running, it prints - some messages telling which features it is checking for. - - 2. Type `make' to compile the package. - - 3. Optionally, type `make check' to run any self-tests that come with - the package, generally using the just-built uninstalled binaries. - - 4. Type `make install' to install the programs and any data files and - documentation. When installing into a prefix owned by root, it is - recommended that the package be configured and built as a regular - user, and only the `make install' phase executed with root - privileges. - - 5. Optionally, type `make installcheck' to repeat any self-tests, but - this time using the binaries in their final installed location. - This target does not install anything. Running this target as a - regular user, particularly if the prior `make install' required - root privileges, verifies that the installation completed - correctly. - - 6. You can remove the program binaries and object files from the - source code directory by typing `make clean'. To also remove the - files that `configure' created (so you can compile the package for - a different kind of computer), type `make distclean'. There is - also a `make maintainer-clean' target, but that is intended mainly - for the package's developers. If you use it, you may have to get - all sorts of other programs in order to regenerate files that came - with the distribution. - - 7. Often, you can also type `make uninstall' to remove the installed - files again. In practice, not all packages have tested that - uninstallation works correctly, even though it is required by the - GNU Coding Standards. - - 8. Some packages, particularly those that use Automake, provide `make - distcheck', which can by used by developers to test that all other - targets like `make install' and `make uninstall' work correctly. - This target is generally not run by end users. - -Compilers and Options -===================== - - Some systems require unusual options for compilation or linking that -the `configure' script does not know about. Run `./configure --help' -for details on some of the pertinent environment variables. - - You can give `configure' initial values for configuration parameters -by setting variables in the command line or in the environment. Here -is an example: - - ./configure CC=c99 CFLAGS=-g LIBS=-lposix - - *Note Defining Variables::, for more details. - -Compiling For Multiple Architectures -==================================== - - You can compile the package for more than one kind of computer at the -same time, by placing the object files for each architecture in their -own directory. To do this, you can use GNU `make'. `cd' to the -directory where you want the object files and executables to go and run -the `configure' script. `configure' automatically checks for the -source code in the directory that `configure' is in and in `..'. This -is known as a "VPATH" build. - - With a non-GNU `make', it is safer to compile the package for one -architecture at a time in the source code directory. After you have -installed the package for one architecture, use `make distclean' before -reconfiguring for another architecture. - - On MacOS X 10.5 and later systems, you can create libraries and -executables that work on multiple system types--known as "fat" or -"universal" binaries--by specifying multiple `-arch' options to the -compiler but only a single `-arch' option to the preprocessor. Like -this: - - ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CPP="gcc -E" CXXCPP="g++ -E" - - This is not guaranteed to produce working output in all cases, you -may have to build one architecture at a time and combine the results -using the `lipo' tool if you have problems. - -Installation Names -================== - - By default, `make install' installs the package's commands under -`/usr/local/bin', include files under `/usr/local/include', etc. You -can specify an installation prefix other than `/usr/local' by giving -`configure' the option `--prefix=PREFIX', where PREFIX must be an -absolute file name. - - You can specify separate installation prefixes for -architecture-specific files and architecture-independent files. If you -pass the option `--exec-prefix=PREFIX' to `configure', the package uses -PREFIX as the prefix for installing programs and libraries. -Documentation and other data files still use the regular prefix. - - In addition, if you use an unusual directory layout you can give -options like `--bindir=DIR' to specify different values for particular -kinds of files. Run `configure --help' for a list of the directories -you can set and what kinds of files go in them. In general, the -default for these options is expressed in terms of `${prefix}', so that -specifying just `--prefix' will affect all of the other directory -specifications that were not explicitly provided. - - The most portable way to affect installation locations is to pass the -correct locations to `configure'; however, many packages provide one or -both of the following shortcuts of passing variable assignments to the -`make install' command line to change installation locations without -having to reconfigure or recompile. - - The first method involves providing an override variable for each -affected directory. For example, `make install -prefix=/alternate/directory' will choose an alternate location for all -directory configuration variables that were expressed in terms of -`${prefix}'. Any directories that were specified during `configure', -but not in terms of `${prefix}', must each be overridden at install -time for the entire installation to be relocated. The approach of -makefile variable overrides for each directory variable is required by -the GNU Coding Standards, and ideally causes no recompilation. -However, some platforms have known limitations with the semantics of -shared libraries that end up requiring recompilation when using this -method, particularly noticeable in packages that use GNU Libtool. - - The second method involves providing the `DESTDIR' variable. For -example, `make install DESTDIR=/alternate/directory' will prepend -`/alternate/directory' before all installation names. The approach of -`DESTDIR' overrides is not required by the GNU Coding Standards, and -does not work on platforms that have drive letters. On the other hand, -it does better at avoiding recompilation issues, and works well even -when some directory options were not specified in terms of `${prefix}' -at `configure' time. - -Optional Features -================= - - If the package supports it, you can cause programs to be installed -with an extra prefix or suffix on their names by giving `configure' the -option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. - - Some packages pay attention to `--enable-FEATURE' options to -`configure', where FEATURE indicates an optional part of the package. -They may also pay attention to `--with-PACKAGE' options, where PACKAGE -is something like `gnu-as' or `x' (for the X Window System). The -`README' should mention any `--enable-' and `--with-' options that the -package recognizes. - - For packages that use the X Window System, `configure' can usually -find the X include and library files automatically, but if it doesn't, -you can use the `configure' options `--x-includes=DIR' and -`--x-libraries=DIR' to specify their locations. - - Some packages offer the ability to configure how verbose the -execution of `make' will be. For these packages, running `./configure ---enable-silent-rules' sets the default to minimal output, which can be -overridden with `make V=1'; while running `./configure ---disable-silent-rules' sets the default to verbose, which can be -overridden with `make V=0'. - -Particular systems -================== - - On HP-UX, the default C compiler is not ANSI C compatible. If GNU -CC is not installed, it is recommended to use the following options in -order to use an ANSI C compiler: - - ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" - -and if that doesn't work, install pre-built binaries of GCC for HP-UX. - - On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot -parse its `' header file. The option `-nodtk' can be used as -a workaround. If GNU CC is not installed, it is therefore recommended -to try - - ./configure CC="cc" - -and if that doesn't work, try - - ./configure CC="cc -nodtk" - - On Solaris, don't put `/usr/ucb' early in your `PATH'. This -directory contains several dysfunctional programs; working variants of -these programs are available in `/usr/bin'. So, if you need `/usr/ucb' -in your `PATH', put it _after_ `/usr/bin'. - - On Haiku, software installed for all users goes in `/boot/common', -not `/usr/local'. It is recommended to use the following options: - - ./configure --prefix=/boot/common - -Specifying the System Type -========================== - - There may be some features `configure' cannot figure out -automatically, but needs to determine by the type of machine the package -will run on. Usually, assuming the package is built to be run on the -_same_ architectures, `configure' can figure that out, but if it prints -a message saying it cannot guess the machine type, give it the -`--build=TYPE' option. TYPE can either be a short name for the system -type, such as `sun4', or a canonical name which has the form: - - CPU-COMPANY-SYSTEM - -where SYSTEM can have one of these forms: - - OS - KERNEL-OS - - See the file `config.sub' for the possible values of each field. If -`config.sub' isn't included in this package, then this package doesn't -need to know the machine type. - - If you are _building_ compiler tools for cross-compiling, you should -use the option `--target=TYPE' to select the type of system they will -produce code for. - - If you want to _use_ a cross compiler, that generates code for a -platform different from the build platform, you should specify the -"host" platform (i.e., that on which the generated programs will -eventually be run) with `--host=TYPE'. - -Sharing Defaults -================ - - If you want to set default values for `configure' scripts to share, -you can create a site shell script called `config.site' that gives -default values for variables like `CC', `cache_file', and `prefix'. -`configure' looks for `PREFIX/share/config.site' if it exists, then -`PREFIX/etc/config.site' if it exists. Or, you can set the -`CONFIG_SITE' environment variable to the location of the site script. -A warning: not all `configure' scripts look for a site script. - -Defining Variables -================== - - Variables not defined in a site shell script can be set in the -environment passed to `configure'. However, some packages may run -configure again during the build, and the customized values of these -variables may be lost. In order to avoid this problem, you should set -them in the `configure' command line, using `VAR=value'. For example: - - ./configure CC=/usr/local2/bin/gcc - -causes the specified `gcc' to be used as the C compiler (unless it is -overridden in the site shell script). - -Unfortunately, this technique does not work for `CONFIG_SHELL' due to -an Autoconf bug. Until the bug is fixed you can use this workaround: - - CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash - -`configure' Invocation -====================== - - `configure' recognizes the following options to control how it -operates. - -`--help' -`-h' - Print a summary of all of the options to `configure', and exit. - -`--help=short' -`--help=recursive' - Print a summary of the options unique to this package's - `configure', and exit. The `short' variant lists options used - only in the top level, while the `recursive' variant lists options - also present in any nested packages. - -`--version' -`-V' - Print the version of Autoconf used to generate the `configure' - script, and exit. - -`--cache-file=FILE' - Enable the cache: use and save the results of the tests in FILE, - traditionally `config.cache'. FILE defaults to `/dev/null' to - disable caching. - -`--config-cache' -`-C' - Alias for `--cache-file=config.cache'. - -`--quiet' -`--silent' -`-q' - Do not print messages saying which checks are being made. To - suppress all normal output, redirect it to `/dev/null' (any error - messages will still be shown). - -`--srcdir=DIR' - Look for the package's source code in directory DIR. Usually - `configure' can determine that directory automatically. - -`--prefix=DIR' - Use DIR as the installation prefix. *note Installation Names:: - for more details, including other options available for fine-tuning - the installation locations. - -`--no-create' -`-n' - Run the configure checks, but stop before creating any output - files. - -`configure' also accepts some other, not widely useful, options. Run -`configure --help' for more details. - diff --git a/README.md b/README.md index 1a4c10148..4fe73476a 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,13 @@ Hydra is a [Continuous Integration](https://en.wikipedia.org/wiki/Continuous_int ## Installation And Setup -**Note**: The instructions provided below are intended to enable new users to get a simple, local installation up and running. They are by no means sufficient for running a production server, let alone a public instance. +**Note**: The instructions provided below are intended to enable new users to get a simple, local installation up and running. +They are by no means sufficient for running a production server, let alone a public instance. ### Enabling The Service -Running Hydra is currently only supported on NixOS. The [hydra module](https://github.com/NixOS/nixpkgs/blob/release-20.03/nixos/modules/services/continuous-integration/hydra/default.nix) allows for an easy setup. The following configuration can be used for a simple setup that performs all builds on _localhost_ (Please refer to the [Options page](https://nixos.org/nixos/options.html#services.hydra) for all available options): +Running Hydra is currently only supported on NixOS. +The [hydra module](https://github.com/NixOS/nixpkgs/blob/release-20.03/nixos/modules/services/continuous-integration/hydra/default.nix) allows for an easy setup. +The following configuration can be used for a simple setup that performs all builds on _localhost_ (Please refer to the [Options page](https://nixos.org/nixos/options.html#services.hydra) for all available options): ```nix { @@ -22,7 +25,8 @@ Running Hydra is currently only supported on NixOS. The [hydra module](https://g } ``` ### Creating An Admin User -Once the Hydra service has been configured as above and activated, you should already be able to access the UI interface at the specified URL. However some actions require an admin user which has to be created first: +Once the Hydra service has been configured as above and activated, you should already be able to access the UI interface at the specified URL. +However some actions require an admin user which has to be created first: ``` $ su - hydra @@ -30,10 +34,13 @@ $ hydra-create-user --full-name '' \ --email-address '' --password-prompt --role admin ``` -Afterwards you should be able to log by clicking on "_Sign In_" on the top right of the web interface using the credentials specified by `hydra-create-user`. Once you are logged in you can click "_Admin -> Create Project_" to configure your first project. +Afterwards you should be able to log in by clicking on "_Sign In_" on the top right of the web interface using the credentials specified by `hydra-create-user`. +Once you are logged in you can click "_Admin -> Create Project_" to configure your first project. ### Creating A Simple Project And Jobset -In order to evaluate and build anything you need to create _projects_ that contain _jobsets_. Hydra supports imperative and declarative projects and many different configurations. The steps below will guide you through the required steps to creating a minimal imperative project configuration. +In order to evaluate and build anything you need to create _projects_ that contain _jobsets_. +Hydra supports imperative and declarative projects and many different configurations. +The steps below will guide you through the required steps to creating a minimal imperative project configuration. #### Creating A Project Log in as administrator, click "_Admin_" and select "_Create project_". Fill the form as follows: @@ -45,7 +52,10 @@ Log in as administrator, click "_Admin_" and select "_Create project_". Fill the Click "_Create project_". #### Creating A Jobset -After creating a project you are forwarded to the project page. Click "_Actions_" and choose "_Create jobset_". Change **Type** to Legacy for the example below. Fill the form with the following values: +After creating a project you are forwarded to the project page. +Click "_Actions_" and choose "_Create jobset_". +Change **Type** to Legacy for the example below. +Fill the form with the following values: - **Identifier**: `hello-project` - **Nix expression**: `examples/hello.nix` in `hydra` @@ -62,7 +72,9 @@ We have to add two inputs for this jobset. One for _nixpkgs_ and one for _hydra_ - **Type**: `Git checkout` - **Value**: `https://github.com/nixos/hydra` -Make sure **State** at the top of the page is set to "_Enabled_" and click on "_Create jobset_". This concludes the creation of a jobset that evaluates [./examples/hello.nix](./examples/hello.nix) once a minute. Clicking "_Evaluations_" should list the first evaluation of the newly created jobset after a brief delay. +Make sure **State** at the top of the page is set to "_Enabled_" and click on "_Create jobset_". +This concludes the creation of a jobset that evaluates [./examples/hello.nix](./examples/hello.nix) once a minute. +Clicking "_Evaluations_" should list the first evaluation of the newly created jobset after a brief delay. ## Building And Developing @@ -88,8 +100,8 @@ The development environment can also automatically be established using [nix-dir ### Executing Hydra During Development -When working on new features or bug fixes you need to be able to run Hydra from your working copy. This -can be done using [foreman](https://github.com/ddollar/foreman): +When working on new features or bug fixes you need to be able to run Hydra from your working copy. +This can be done using [foreman](https://github.com/ddollar/foreman): ``` $ nix develop @@ -99,14 +111,14 @@ $ cargo build $ foreman start ``` -Have a look at the [Procfile](./Procfile) if you want to see how the processes are being started. In order to avoid -conflicts with services that might be running on your host, hydra and postgress are started on custom ports: +Have a look at the [Procfile](./Procfile) if you want to see how the processes are being started. +In order to avoid conflicts with services that might be running on your host, hydra and postgress are started on custom ports: - hydra-server: 63333 with the username "alice" and the password "foobar" - postgresql: 64444, can be connected to using `psql -p 64444 -h localhost hydra` -Note that this is only ever meant as an ad-hoc way of executing Hydra during development. Please make use of the -NixOS module for actually running Hydra in production. +Note that this is only ever meant as an ad-hoc way of executing Hydra during development. +Please make use of the NixOS module for actually running Hydra in production. ### Checking your patches @@ -146,7 +158,8 @@ $ cargo test ### JSON API -You can also interface with Hydra through a JSON API. The API is defined in [hydra-api.yaml](./hydra-api.yaml) and you can test and explore via the [swagger editor](https://editor.swagger.io/?url=https://raw.githubusercontent.com/NixOS/hydra/master/hydra-api.yaml) +You can also interface with Hydra through a JSON API. +The API is defined in [hydra-api.yaml](./hydra-api.yaml) and you can test and explore via the [swagger editor](https://editor.swagger.io/?url=https://raw.githubusercontent.com/NixOS/hydra/master/hydra-api.yaml) ## Additional Resources diff --git a/clippy.toml b/clippy.toml index e8389aaf9..25cf7fb46 100644 --- a/clippy.toml +++ b/clippy.toml @@ -22,6 +22,10 @@ disallowed-methods = [ { path = "std::fs::File::create_new", replacement = "fs_err::File::create_new" }, { path = "std::fs::OpenOptions::new", replacement = "fs_err::OpenOptions::new" }, + { path = "std::ffi::OsStr::to_string_lossy", reason = "Handle UTF-8 errors" }, + + { path = "std::string::String::from_utf8_lossy", reason = "Handle UTF-8 errors" }, + { path = "std::os::unix::fs::symlink", replacement = "fs_err::os::unix::symlink" }, { path = "std::path::Path::try_exists", reason = "Use fs_err::path::PathExt methods" }, @@ -30,6 +34,7 @@ disallowed-methods = [ { path = "std::path::Path::canonicalize", reason = "Use fs_err::path::PathExt methods" }, { path = "std::path::Path::read_link", reason = "Use fs_err::path::PathExt methods" }, { path = "std::path::Path::read_dir", reason = "Use fs_err::path::PathExt methods" }, + { path = "std::path::Path::to_string_lossy", reason = "Handle UTF-8 errors" }, { path = "tokio::fs::canonicalize", replacement = "fs_err::tokio::canonicalize" }, { path = "tokio::fs::copy", replacement = "fs_err::tokio::copy" }, diff --git a/flake.lock b/flake.lock index 35cfbaa7e..cb9a3b9f6 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ "nix": { "flake": false, "locked": { - "lastModified": 1777921551, - "narHash": "sha256-uioYvCRbqKsAC+wKbNoNe27TyXUUeFgevv4FJ+ITzWY=", + "lastModified": 1779318684, + "narHash": "sha256-zj8d2tTd3DGiVKMMjuIUpMODgjJfxWP4hkQ8bdhHQYk=", "owner": "NixOS", "repo": "nix", - "rev": "24a072442bee6ad4c3a2da92abafff987881da26", + "rev": "8294f1c1fda17f87e65723ac9348395d2f07513c", "type": "github" }, "original": { @@ -53,18 +53,15 @@ }, "nixpkgs": { "locked": { - "lastModified": 1777933422, - "narHash": "sha256-WpfZLatstZKZV43HtpK4cbEdQZvHhwVRVtW/rc6YTRU=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "41a7bd591bc47320390c829b9f16bfc59b25843c", - "type": "github" + "lastModified": 1780453794, + "narHash": "sha256-hhAl/iKiurXPn7rdzDgiSuRB8tqOB6f0buWkh8Y9mkY=", + "rev": "6b316287bae2ee04c9b93c8c858d930fd07d7338", + "type": "tarball", + "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.1183.6b316287bae2/nixexprs.tar.xz" }, "original": { - "owner": "NixOS", - "ref": "nixos-25.11-small", - "repo": "nixpkgs", - "type": "github" + "type": "tarball", + "url": "https://channels.nixos.org/nixos-26.05/nixexprs.tar.xz" } }, "root": { @@ -83,11 +80,11 @@ ] }, "locked": { - "lastModified": 1775636079, - "narHash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0=", + "lastModified": 1780220602, + "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "790751ff7fd3801feeaf96d7dc416a8d581265ba", + "rev": "db947814a175b7ca6ded66e21383d938df01c227", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 51a0be014..0dfb1b9a4 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "A Nix-based continuous build system"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11-small"; + inputs.nixpkgs.url = "https://channels.nixos.org/nixos-26.05/nixexprs.tar.xz"; inputs.nix = { url = "github:NixOS/nix/2.34-maintenance"; @@ -78,10 +78,8 @@ hydra-linters = self'.callPackage ./subprojects/hydra-linters/package.nix { }; hydra-queue-runner = self'.callPackage ./subprojects/hydra-queue-runner/package.nix { - inherit nixComponents; }; hydra-builder = self'.callPackage ./subprojects/hydra-builder/package.nix { - inherit nixComponents; }; }); mkHydraBuilder = @@ -89,7 +87,6 @@ pkgs.lib.makeScope pkgs.newScope (self': { inherit version releaseVersion; hydra-builder = self'.callPackage ./subprojects/hydra-builder/package.nix { - inherit nixComponents; }; }); @@ -153,7 +150,7 @@ builder = forEachSystemIncDarwin (system: packages.${system}.hydra-builder); - nixosTests = import ./nixos-tests.nix { + nixosTests = import ./nixos-tests { inherit forEachSystem nixpkgs nixosModules; }; @@ -189,6 +186,9 @@ inherit (packages.${system}) hydra; }; formatter = (treefmtEval system).config.build.check self; + dependency-diagram = pkgs.callPackage ./packaging/check-dependency-diagram.nix { + src = self; + }; } ); diff --git a/nixos-tests.nix b/nixos-tests.nix deleted file mode 100644 index e8f345086..000000000 --- a/nixos-tests.nix +++ /dev/null @@ -1,371 +0,0 @@ -{ - forEachSystem, - nixpkgs, - nixosModules, -}: - -let - # Shared nix settings for all test VMs - nixSettings = { - settings.substituters = [ ]; - }; - - serverConfig = - { pkgs, ... }: - { - imports = [ - nixosModules.web-app - nixosModules.queue-runner - ]; - - services.hydra-dev.enable = true; - services.hydra-dev.hydraURL = "http://hydra.example.org"; - services.hydra-dev.notificationSender = "admin@hydra.example.org"; - - services.hydra-queue-runner-dev.enable = true; - services.hydra-queue-runner-dev.grpc.address = "[::]"; - - systemd.services.hydra-send-stats.enable = false; - - services.postgresql.enable = true; - - time.timeZone = "UTC"; - - nix = nixSettings // { - extraOptions = '' - allowed-uris = https://github.com/ - ''; - }; - - networking.firewall.allowedTCPPorts = [ 50051 ]; - - virtualisation.memorySize = 2048; - virtualisation.writableStore = true; - - environment.systemPackages = [ - pkgs.perlPackages.LWP - pkgs.perlPackages.JSON - ]; - }; - - builderConfig = - { ... }: - { - imports = [ - nixosModules.builder - ]; - - services.hydra-queue-builder-dev.enable = true; - services.hydra-queue-builder-dev.queueRunnerAddr = "http://server:50051"; - - virtualisation.memorySize = 2048; - virtualisation.writableStore = true; - - nix = nixSettings; - }; - -in - -{ - - install = forEachSystem ( - system: - (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest { - name = "hydra-install"; - nodes.server = serverConfig; - nodes.builder = builderConfig; - testScript = '' - server.wait_for_unit("hydra-init.service") - server.succeed("systemctl status hydra-server.socket") - server.wait_for_unit("hydra-server.service") - server.wait_for_unit("hydra-evaluator.service") - server.wait_for_unit("hydra-queue-runner-dev.service") - builder.wait_for_unit("hydra-queue-builder-dev.service") - server.wait_for_open_port(3000) - server.succeed("curl --fail http://localhost:3000/") - ''; - } - ); - - notifications = forEachSystem ( - system: - (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest { - name = "hydra-notifications"; - nodes.server = { - imports = [ serverConfig ]; - services.hydra-dev.extraConfig = '' - - url = http://127.0.0.1:8086 - db = hydra - - ''; - services.influxdb.enable = true; - }; - nodes.builder = builderConfig; - testScript = - { nodes, ... }: - '' - server.wait_for_unit("hydra-init.service") - server.wait_for_unit("hydra-queue-runner-dev.service") - builder.wait_for_unit("hydra-queue-builder-dev.service") - - # Create an admin account and some other state. - server.succeed( - """ - su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin" - mkdir /run/jobset - chmod 755 /run/jobset - cp ${./subprojects/hydra-tests/jobs/api-test.nix} /run/jobset/default.nix - chmod 644 /run/jobset/default.nix - chown -R hydra /run/jobset - """ - ) - - # Wait until InfluxDB can receive web requests - server.wait_for_unit("influxdb.service") - server.wait_for_open_port(8086) - - # Create an InfluxDB database where hydra will write to - server.succeed( - "curl -XPOST 'http://127.0.0.1:8086/query' " - + "--data-urlencode 'q=CREATE DATABASE hydra'" - ) - - # Wait until hydra-server can receive HTTP requests - server.wait_for_unit("hydra-server.service") - server.wait_for_open_port(3000) - - # Setup the project and jobset - server.succeed( - "su - hydra -c 'perl -I ${nodes.server.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${./subprojects/hydra-tests/setup-notifications-jobset.pl}' >&2" - ) - - # Wait until hydra has build the job and - # the InfluxDBNotification plugin uploaded its notification to InfluxDB - server.wait_until_succeeds( - "curl -s -H 'Accept: application/csv' " - + "-G 'http://127.0.0.1:8086/query?db=hydra' " - + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success" - ) - ''; - } - ); - - gitea = forEachSystem ( - system: - let - pkgs = nixpkgs.legacyPackages.${system}; - in - (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest { - name = "hydra-gitea"; - nodes.server = - { pkgs, ... }: - { - imports = [ serverConfig ]; - services.hydra-dev.extraConfig = '' - - root=d7f16a3412e01a43a414535b16007c6931d3a9c7 - - ''; - nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ]; - services.gitea = { - enable = true; - database.type = "postgres"; - settings = { - service.DISABLE_REGISTRATION = true; - server.HTTP_PORT = 3001; - }; - }; - services.openssh.enable = true; - environment.systemPackages = with pkgs; [ - gitea - git - jq - gawk - ]; - networking.firewall.allowedTCPPorts = [ 3000 ]; - }; - nodes.builder = builderConfig; - skipLint = true; - testScript = - let - scripts.mktoken = pkgs.writeText "token.sql" '' - INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all'); - ''; - - scripts.git-setup = pkgs.writeShellScript "setup.sh" '' - set -x - mkdir -p /tmp/repo $HOME/.ssh - cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk - chmod 0400 $HOME/.ssh/privk - git -C /tmp/repo init - cp ${smallDrv} /tmp/repo/jobset.nix - git -C /tmp/repo add . - git config --global user.email test@localhost - git config --global user.name test - git -C /tmp/repo commit -m 'Initial import' - git -C /tmp/repo remote add origin gitea@server:root/repo - GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \ - git -C /tmp/repo push origin master - git -C /tmp/repo log >&2 - ''; - - scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" '' - set -x - su -l hydra -c "hydra-create-user root --email-address \ - 'alice@example.org' --password foobar --role admin" - - URL=http://localhost:3000 - USERNAME="root" - PASSWORD="foobar" - PROJECT_NAME="trivial" - JOBSET_NAME="trivial" - mycurl() { - curl --referer $URL -H "Accept: application/json" \ - -H "Content-Type: application/json" $@ - } - - cat >data.json <data.json <data.json < $out; exit 0"]; - }; - } - ''; - in - '' - import json - - server.start() - builder.start() - server.wait_for_unit("multi-user.target") - server.wait_for_unit("hydra-queue-runner-dev.service") - builder.wait_for_unit("hydra-queue-builder-dev.service") - server.wait_for_open_port(3000) - server.wait_for_open_port(3001) - - server.succeed( - "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create " - + "--username root --password root --email test@localhost'" - ) - server.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'") - - server.succeed( - "curl --fail -X POST http://localhost:3001/api/v1/user/repos " - + "-H 'Accept: application/json' -H 'Content-Type: application/json' " - + f"-H 'Authorization: token ${api_token}'" - + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\''' - ) - - server.succeed( - "curl --fail -X POST http://localhost:3001/api/v1/user/keys " - + "-H 'Accept: application/json' -H 'Content-Type: application/json' " - + f"-H 'Authorization: token ${api_token}'" - + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\''' - ) - - server.succeed( - "${scripts.git-setup}" - ) - - server.succeed( - "${scripts.hydra-setup}" - ) - - server.wait_until_succeeds( - 'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" ' - + '| jq .buildstatus | xargs test 0 -eq' - ) - - data = server.succeed( - 'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" ' - + "-H 'Accept: application/json' -H 'Content-Type: application/json' " - + f"-H 'Authorization: token ${api_token}'" - ) - - response = json.loads(data) - - assert len(response) == 2, "Expected exactly two status updates for latest commit (queued, finished)!" - items = {item['status'] for item in response} - assert items == {"success", "pending"}, "Expected one success status and one pending status" - - server.shutdown() - ''; - } - ); - - validate-openapi = forEachSystem ( - system: - let - pkgs = nixpkgs.legacyPackages.${system}; - in - pkgs.runCommand "validate-openapi" { buildInputs = [ pkgs.openapi-generator-cli ]; } '' - openapi-generator-cli validate -i ${./hydra-api.yaml} - touch $out - '' - ); - -} diff --git a/nixos-tests/common.nix b/nixos-tests/common.nix new file mode 100644 index 000000000..fef375222 --- /dev/null +++ b/nixos-tests/common.nix @@ -0,0 +1,66 @@ +{ + nixosModules, +}: + +let + # Shared nix settings for all test VMs + nixSettings = { + settings.substituters = [ ]; + }; +in + +{ + serverConfig = + { pkgs, ... }: + { + imports = [ + nixosModules.web-app + nixosModules.queue-runner + ]; + + services.hydra-dev.enable = true; + services.hydra-dev.hydraURL = "http://hydra.example.org"; + services.hydra-dev.notificationSender = "admin@hydra.example.org"; + + services.hydra-queue-runner-dev.enable = true; + services.hydra-queue-runner-dev.grpc.address = "[::]"; + + systemd.services.hydra-send-stats.enable = false; + + services.postgresql.enable = true; + + time.timeZone = "UTC"; + + nix = nixSettings // { + extraOptions = '' + allowed-uris = https://github.com/ + ''; + }; + + networking.firewall.allowedTCPPorts = [ 50051 ]; + + virtualisation.memorySize = 2048; + virtualisation.writableStore = true; + + environment.systemPackages = [ + pkgs.perlPackages.LWP + pkgs.perlPackages.JSON + ]; + }; + + builderConfig = + { ... }: + { + imports = [ + nixosModules.builder + ]; + + services.hydra-queue-builder-dev.enable = true; + services.hydra-queue-builder-dev.queueRunnerAddr = "http://server:50051"; + + virtualisation.memorySize = 2048; + virtualisation.writableStore = true; + + nix = nixSettings; + }; +} diff --git a/nixos-tests/default.nix b/nixos-tests/default.nix new file mode 100644 index 000000000..508950d10 --- /dev/null +++ b/nixos-tests/default.nix @@ -0,0 +1,36 @@ +{ + forEachSystem, + nixpkgs, + nixosModules, +}: + +let + common = import ./common.nix { inherit nixosModules; }; +in + +{ + + install = forEachSystem (system: import ./install.nix { inherit system nixpkgs common; }); + + notifications = forEachSystem ( + system: import ./notifications.nix { inherit system nixpkgs common; } + ); + + gitea = forEachSystem (system: import ./gitea.nix { inherit system nixpkgs common; }); + + s3-nar-listing = forEachSystem ( + system: import ./s3-nar-listing.nix { inherit system nixpkgs common; } + ); + + validate-openapi = forEachSystem ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + pkgs.runCommand "validate-openapi" { buildInputs = [ pkgs.openapi-generator-cli ]; } '' + openapi-generator-cli validate -i ${../hydra-api.yaml} + touch $out + '' + ); + +} diff --git a/nixos-tests/gitea.nix b/nixos-tests/gitea.nix new file mode 100644 index 000000000..b503b8544 --- /dev/null +++ b/nixos-tests/gitea.nix @@ -0,0 +1,207 @@ +{ + system, + nixpkgs, + common, +}: + +let + pkgs = nixpkgs.legacyPackages.${system}; + + api_token = "d7f16a3412e01a43a414535b16007c6931d3a9c7"; + + snakeoilKeypair = { + privkey = pkgs.writeText "privkey.snakeoil" '' + -----BEGIN EC PRIVATE KEY----- + MHcCAQEEIHQf/khLvYrQ8IOika5yqtWvI0oquHlpRLTZiJy5dRJmoAoGCCqGSM49 + AwEHoUQDQgAEKF0DYGbBwbj06tA3fd/+yP44cvmwmHBWXZCKbS+RQlAKvLXMWkpN + r1lwMyJZoSGgBHoUahoYjTh9/sJL7XLJtA== + -----END EC PRIVATE KEY----- + ''; + + pubkey = pkgs.lib.concatStrings [ + "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHA" + "yNTYAAABBBChdA2BmwcG49OrQN33f/sj+OHL5sJhwVl2Qim0vkUJQCry1zFpKTa" + "9ZcDMiWaEhoAR6FGoaGI04ff7CS+1yybQ= sakeoil" + ]; + }; + + scripts.mktoken = pkgs.writeText "token.sql" '' + INSERT INTO access_token (id, uid, name, created_unix, updated_unix, token_hash, token_salt, token_last_eight, scope) VALUES (1, 1, 'hydra', 1617107360, 1617107360, 'a930f319ca362d7b49a4040ac0af74521c3a3c3303a86f327b01994430672d33b6ec53e4ea774253208686c712495e12a486', 'XRjWE9YW0g', '31d3a9c7', 'all'); + ''; + + scripts.git-setup = pkgs.writeShellScript "setup.sh" '' + set -x + mkdir -p /tmp/repo $HOME/.ssh + cat ${snakeoilKeypair.privkey} > $HOME/.ssh/privk + chmod 0400 $HOME/.ssh/privk + git -C /tmp/repo init + cp ${smallDrv} /tmp/repo/jobset.nix + git -C /tmp/repo add . + git config --global user.email test@localhost + git config --global user.name test + git -C /tmp/repo commit -m 'Initial import' + git -C /tmp/repo remote add origin gitea@server:root/repo + GIT_SSH_COMMAND='ssh -i $HOME/.ssh/privk -o StrictHostKeyChecking=no' \ + git -C /tmp/repo push origin master + git -C /tmp/repo log >&2 + ''; + + scripts.hydra-setup = pkgs.writeShellScript "hydra.sh" '' + set -x + su -l hydra -c "hydra-create-user root --email-address \ + 'alice@example.org' --password foobar --role admin" + + URL=http://localhost:3000 + USERNAME="root" + PASSWORD="foobar" + PROJECT_NAME="trivial" + JOBSET_NAME="trivial" + mycurl() { + curl --referer $URL -H "Accept: application/json" \ + -H "Content-Type: application/json" $@ + } + + cat >data.json <data.json <data.json < $out; exit 0"]; + }; + } + ''; +in + +(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest { + name = "hydra-gitea"; + nodes.server = + { pkgs, ... }: + { + imports = [ common.serverConfig ]; + services.hydra-dev.extraConfig = '' + + root=d7f16a3412e01a43a414535b16007c6931d3a9c7 + + ''; + nixpkgs.config.permittedInsecurePackages = [ "gitea-1.19.4" ]; + services.gitea = { + enable = true; + database.type = "postgres"; + settings = { + service.DISABLE_REGISTRATION = true; + server.HTTP_PORT = 3001; + }; + }; + services.openssh.enable = true; + environment.systemPackages = with pkgs; [ + gitea + git + jq + gawk + ]; + networking.firewall.allowedTCPPorts = [ 3000 ]; + }; + nodes.builder = common.builderConfig; + skipLint = true; + testScript = '' + import json + + server.start() + builder.start() + server.wait_for_unit("multi-user.target") + server.wait_for_unit("hydra-queue-runner-dev.service") + builder.wait_for_unit("hydra-queue-builder-dev.service") + server.wait_for_open_port(3000) + server.wait_for_open_port(3001) + + server.succeed( + "su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create " + + "--username root --password root --email test@localhost'" + ) + server.succeed("su -l postgres -c 'psql gitea < ${scripts.mktoken}'") + + server.succeed( + "curl --fail -X POST http://localhost:3001/api/v1/user/repos " + + "-H 'Accept: application/json' -H 'Content-Type: application/json' " + + f"-H 'Authorization: token ${api_token}'" + + ' -d \'{"auto_init":false, "description":"string", "license":"mit", "name":"repo", "private":false}\''' + ) + + server.succeed( + "curl --fail -X POST http://localhost:3001/api/v1/user/keys " + + "-H 'Accept: application/json' -H 'Content-Type: application/json' " + + f"-H 'Authorization: token ${api_token}'" + + ' -d \'{"key":"${snakeoilKeypair.pubkey}","read_only":true,"title":"SSH"}\''' + ) + + server.succeed( + "${scripts.git-setup}" + ) + + server.succeed( + "${scripts.hydra-setup}" + ) + + server.wait_until_succeeds( + 'curl -Lf -s http://localhost:3000/build/1 -H "Accept: application/json" ' + + '| jq .buildstatus | xargs test 0 -eq' + ) + + data = server.succeed( + 'curl -Lf -s "http://localhost:3001/api/v1/repos/root/repo/statuses/$(cd /tmp/repo && git show | head -n1 | awk "{print \\$2}")" ' + + "-H 'Accept: application/json' -H 'Content-Type: application/json' " + + f"-H 'Authorization: token ${api_token}'" + ) + + response = json.loads(data) + + assert len(response) == 2, "Expected exactly two status updates for latest commit (queued, finished)!" + items = {item['status'] for item in response} + assert items == {"success", "pending"}, "Expected one success status and one pending status" + + server.shutdown() + ''; +} diff --git a/nixos-tests/install.nix b/nixos-tests/install.nix new file mode 100644 index 000000000..57fffa4f3 --- /dev/null +++ b/nixos-tests/install.nix @@ -0,0 +1,21 @@ +{ + system, + nixpkgs, + common, +}: + +(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest { + name = "hydra-install"; + nodes.server = common.serverConfig; + nodes.builder = common.builderConfig; + testScript = '' + server.wait_for_unit("hydra-init.service") + server.succeed("systemctl status hydra-server.socket") + server.wait_for_unit("hydra-server.service") + server.wait_for_unit("hydra-evaluator.service") + server.wait_for_unit("hydra-queue-runner-dev.service") + builder.wait_for_unit("hydra-queue-builder-dev.service") + server.wait_for_open_port(3000) + server.succeed("curl --fail http://localhost:3000/") + ''; +} diff --git a/nixos-tests/notifications.nix b/nixos-tests/notifications.nix new file mode 100644 index 000000000..786c337cb --- /dev/null +++ b/nixos-tests/notifications.nix @@ -0,0 +1,66 @@ +{ + system, + nixpkgs, + common, +}: + +(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).simpleTest { + name = "hydra-notifications"; + nodes.server = { + imports = [ common.serverConfig ]; + services.hydra-dev.extraConfig = '' + + url = http://127.0.0.1:8086 + db = hydra + + ''; + services.influxdb.enable = true; + }; + nodes.builder = common.builderConfig; + testScript = + { nodes, ... }: + '' + server.wait_for_unit("hydra-init.service") + server.wait_for_unit("hydra-queue-runner-dev.service") + builder.wait_for_unit("hydra-queue-builder-dev.service") + + # Create an admin account and some other state. + server.succeed( + """ + su - hydra -c "hydra-create-user root --email-address 'alice@example.org' --password foobar --role admin" + mkdir /run/jobset + chmod 755 /run/jobset + cp ${../subprojects/hydra-tests/jobs/api-test.nix} /run/jobset/default.nix + chmod 644 /run/jobset/default.nix + chown -R hydra /run/jobset + """ + ) + + # Wait until InfluxDB can receive web requests + server.wait_for_unit("influxdb.service") + server.wait_for_open_port(8086) + + # Create an InfluxDB database where hydra will write to + server.succeed( + "curl -XPOST 'http://127.0.0.1:8086/query' " + + "--data-urlencode 'q=CREATE DATABASE hydra'" + ) + + # Wait until hydra-server can receive HTTP requests + server.wait_for_unit("hydra-server.service") + server.wait_for_open_port(3000) + + # Setup the project and jobset + server.succeed( + "su - hydra -c 'perl -I ${nodes.server.services.hydra-dev.package.perlDeps}/lib/perl5/site_perl ${../subprojects/hydra-tests/setup-notifications-jobset.pl}' >&2" + ) + + # Wait until hydra has build the job and + # the InfluxDBNotification plugin uploaded its notification to InfluxDB + server.wait_until_succeeds( + "curl -s -H 'Accept: application/csv' " + + "-G 'http://127.0.0.1:8086/query?db=hydra' " + + "--data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success" + ) + ''; +} diff --git a/nixos-tests/s3-nar-listing.nix b/nixos-tests/s3-nar-listing.nix new file mode 100644 index 000000000..8d315a081 --- /dev/null +++ b/nixos-tests/s3-nar-listing.nix @@ -0,0 +1,295 @@ +{ + system, + nixpkgs, + common, +}: + +let + pkgs = nixpkgs.legacyPackages.${system}; + + garagePort = 3900; + garageRpcPort = 3901; + garageAdminPort = 3902; + + # 32-byte hex RPC secret for garage + rpcSecret = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + # A derivation that produces a directory with various entry types + # (regular files, executable files, symlinks, subdirectories) so we + # can verify the NAR listing captures all of them. + # + # The store paths for bash and coreutils are interpolated as literal + # strings by writeText. Inside the expression, builtins.storePath + # re-introduces the string context so that the evaluator on the VM + # knows the derivation depends on them and the sandbox gets access. + jobFile = pkgs.writeText "default.nix" '' + { + trivial = builtins.derivation { + name = "trivial"; + system = "${system}"; + builder = "''${builtins.storePath "${pkgs.bash}"}/bin/bash"; + PATH = "''${builtins.storePath "${pkgs.coreutils}"}/bin"; + allowSubstitutes = false; + preferLocalBuild = true; + args = [ + "-c" + "mkdir -p $out/subdir; echo hello > $out/greeting; echo nested > $out/subdir/file; printf '#!/bin/sh\\necho hi\\n' > $out/run.sh; chmod +x $out/run.sh; ln -s greeting $out/link; exit 0" + ]; + }; + } + ''; + + # The exact NAR listing we expect, used for an exact JSON comparison. + expectedListing = builtins.toJSON { + version = 1; + root = { + type = "directory"; + entries = { + greeting = { + type = "regular"; + executable = false; + size = 6; # "hello\n" + }; + link = { + type = "symlink"; + target = "greeting"; + }; + "run.sh" = { + type = "regular"; + executable = true; + size = 18; # printf "#!/bin/sh\necho hi\n" (no extra trailing newline) + }; + subdir = { + type = "directory"; + entries = { + file = { + type = "regular"; + executable = false; + size = 7; # "nested\n" + }; + }; + }; + }; + }; + }; +in + +(import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest { + name = "hydra-s3"; + + nodes.s3 = + { pkgs, ... }: + { + services.garage = { + enable = true; + package = pkgs.garage; + settings = { + replication_factor = 1; + db_engine = "sqlite"; + rpc_bind_addr = "[::]:${toString garageRpcPort}"; + rpc_secret = rpcSecret; + s3_api = { + s3_region = "garage"; + api_bind_addr = "[::]:${toString garagePort}"; + root_domain = ".s3.garage"; + }; + s3_web = { + bind_addr = "[::]:3903"; + root_domain = ".web.garage"; + }; + admin.api_bind_addr = "[::]:${toString garageAdminPort}"; + }; + }; + + networking.firewall.allowedTCPPorts = [ + garagePort + garageAdminPort + ]; + }; + + nodes.server = + { pkgs, ... }: + { + imports = [ common.serverConfig ]; + + services.hydra-queue-runner-dev = { + settings.remoteStoreAddr = [ + "s3://hydra-cache?endpoint=http://s3:${toString garagePort}®ion=garage&write-nar-listing=1&compression=none&scheme=http" + ]; + awsCredentialsFile = "/var/lib/hydra/queue-runner/.aws-credentials"; + }; + + environment.systemPackages = [ pkgs.jq ]; + }; + + nodes.builder = common.builderConfig; + + skipLint = true; + + testScript = '' + import json + import shlex + + s3.start() + server.start() + builder.start() + + # Wait for garage to start + s3.wait_for_unit("garage.service") + s3.wait_for_open_port(${toString garagePort}) + s3.wait_for_open_port(${toString garageAdminPort}) + + # Configure garage: assign layout, apply, create bucket and key + node_id = s3.succeed("garage node id -q 2>/dev/null").strip() + short_id = node_id.split("@")[0][:16] if "@" in node_id else node_id[:16] + + s3.succeed(f"garage layout assign {short_id} -z dc1 -c 1G") + + version = s3.succeed( + "garage layout show | grep -oP 'apply --version \\K[0-9]+'" + ).strip() + s3.succeed(f"garage layout apply --version {version}") + + s3.succeed("garage bucket create hydra-cache") + + key_output = s3.succeed("garage key create hydra-key") + key_id = "" + key_secret = "" + for line in key_output.splitlines(): + if "Key ID" in line: + key_id = line.split()[-1] + if "Secret key" in line: + key_secret = line.split()[-1] + + s3.succeed( + f"garage bucket allow hydra-cache --read --write --owner --key {key_id}" + ) + + # Write AWS credentials before starting the queue-runner. + # Uses /var/lib/hydra/ (persistent) rather than /run/ (wiped on restart). + creds_path = "/var/lib/hydra/queue-runner/.aws-credentials" + server.wait_for_unit("hydra-init.service") + server.succeed( + f"mkdir -p /var/lib/hydra/queue-runner && " + f"printf '[default]\\naws_access_key_id = {key_id}\\naws_secret_access_key = {key_secret}\\n' > {creds_path} && " + f"chown hydra-queue-runner:hydra {creds_path} && " + f"chmod 600 {creds_path}" + ) + server.succeed("systemctl restart hydra-queue-runner-dev.service") + server.wait_for_unit("hydra-queue-runner-dev.service") + builder.wait_for_unit("hydra-queue-builder-dev.service") + + # Create an admin account and project + server.succeed( + 'su - hydra -c "hydra-create-user root --email-address root@example.org --password foobar --role admin"' + ) + + server.wait_for_unit("hydra-server.service") + server.wait_for_open_port(3000) + + # Create project and jobset via the API + server.succeed( + "mkdir -p /run/jobset && " + "cp ${jobFile} /run/jobset/default.nix && " + "chmod -R 755 /run/jobset && " + "chown -R hydra /run/jobset" + ) + + URL = "http://localhost:3000" + cookie_jar = "/tmp/hydra-cookie.txt" + + def mycurl(method, path, data=None): + cmd = f"curl --referer {shlex.quote(URL)} -H 'Accept: application/json' -H 'Content-Type: application/json'" + cmd += f" -X {method} {shlex.quote(URL + path)}" + cmd += f" -b {cookie_jar} -c {cookie_jar}" + if data: + cmd += f" -d {shlex.quote(json.dumps(data))}" + return server.succeed(cmd) + + mycurl("POST", "/login", { + "username": "root", + "password": "foobar", + }) + + mycurl("PUT", "/project/test", { + "displayname": "Test", + "enabled": "1", + "visible": "1", + }) + + mycurl("PUT", "/jobset/test/trivial", { + "description": "Trivial", + "checkinterval": "0", + "enabled": "1", + "visible": "1", + "keepnr": "1", + "type": 0, + "nixexprinput": "src", + "nixexprpath": "default.nix", + "inputs": { + "src": {"value": "/run/jobset", "type": "path"}, + }, + }) + + # Trigger evaluation + mycurl("POST", "/api/push?jobsets=test:trivial&force=1") + + # Wait for the build to finish (any status — fail fast instead of hanging) + server.wait_until_succeeds( + f'curl -sf {URL}/build/1 -H "Accept: application/json"' + ' | jq -e ".finished == 1"', + timeout=120, + ) + + # Check build succeeded + build_info = json.loads( + server.succeed( + f'curl -sf {URL}/build/1 -H "Accept: application/json"' + ) + ) + if build_info.get("buildstatus") != 0: + drv = build_info.get("drvpath", "unknown") + # Dump the nix build log from inside the builder VM + print(builder.succeed(f"nix-store -l {drv} 2>&1 || true")) + # Also dump hydra's own build log + print( + server.succeed( + f"find /var/lib/hydra/build-logs -type f -exec bzcat {{}} \\; 2>/dev/null || true" + ) + ) + raise Exception( + f"Build failed with status {build_info.get('buildstatus')}, drv={drv}" + ) + + out_path = build_info["buildoutputs"]["out"]["path"] + store_hash = out_path.split("/")[-1][:32] + + # Wait for the .ls listing to appear in S3 (upload may still be in progress) + server.wait_until_succeeds( + f"curl -sf http://s3:${toString garagePort}/hydra-cache/{store_hash}.ls" + f" --aws-sigv4 'aws:amz:garage:s3'" + f" -u '{key_id}:{key_secret}'", + timeout=60, + ) + + # Fetch the .ls listing + ls_json = server.succeed( + f"curl -sf http://s3:${toString garagePort}/hydra-cache/{store_hash}.ls" + f" --aws-sigv4 'aws:amz:garage:s3'" + f" -u '{key_id}:{key_secret}'" + ) + + # Exact comparison with the expected listing + expected = json.loads('${expectedListing}') + actual = json.loads(ls_json) + assert actual == expected, ( + f"NAR listing mismatch.\n" + f"Expected:\n{json.dumps(expected, indent=2)}\n" + f"Actual:\n{json.dumps(actual, indent=2)}" + ) + + builder.shutdown() + server.shutdown() + s3.shutdown() + ''; +} diff --git a/packaging/cargo-output-hashes.nix b/packaging/cargo-output-hashes.nix new file mode 100644 index 000000000..59918a0dc --- /dev/null +++ b/packaging/cargo-output-hashes.nix @@ -0,0 +1,5 @@ +# Shared outputHashes for cargoLock across all Rust crates. +# Update this file when the harmonia dependency changes. +{ + "harmonia-store-derivation-0.0.0-alpha.0" = "sha256-316CfR32JYa6aD8i3Xwlvgdkk0Lq6K+QzCkuitnYAXk="; +} diff --git a/packaging/check-dependency-diagram.nix b/packaging/check-dependency-diagram.nix new file mode 100644 index 000000000..1bc36136f --- /dev/null +++ b/packaging/check-dependency-diagram.nix @@ -0,0 +1,43 @@ +{ + lib, + runCommand, + diffutils, + python3, + cargo, + src, +}: +let + script = ../scripts/dependency-diagram.py; + doc = ../subprojects/hydra-manual/src/architecture.md; + generated = + runCommand "generate-dependency-diagram" + { + nativeBuildInputs = [ + python3 + cargo + ]; + inherit src script doc; + } + '' + python3 "$script" \ + --doc "$doc" \ + --manifest-path $src/Cargo.toml \ + > "$out" + ''; +in +runCommand "check-dependency-diagram" + { + nativeBuildInputs = [ diffutils ]; + } + '' + if ! diff \ + --unified \ + --color=always \ + ${doc} \ + ${generated}; then + echo "Dependency diagram is out of date. Update with:" + echo " cp ${generated} subprojects/hydra-manual/src/architecture.md" + exit 1 + fi + touch $out + '' diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index efafd1973..8bdfc7448 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -47,6 +47,7 @@ hydra.overrideAttrs ( nativeBuildInputs = collectInputs "nativeBuildInputs" ++ [ foreman + pkgs.cargo-nextest pkgs.clippy pkgs.nixfmt pkgs.rustfmt diff --git a/scripts/dependency-diagram.py b/scripts/dependency-diagram.py new file mode 100644 index 000000000..54fa1854d --- /dev/null +++ b/scripts/dependency-diagram.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +"""Generate a Mermaid transitive reduction of intra-workspace crate dependencies.""" + +import json +import re +import subprocess +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +ROOT = SCRIPT_DIR.parent +DEFAULT_DOC_FILE = ROOT / "subprojects" / "hydra-manual" / "src" / "architecture.md" +PREFIX = "" + + +def short(name: str) -> str: + """Shorten crate names for readable node labels.""" + # Hydra crates don't have a uniform prefix, so just return as-is. + return name.removeprefix(PREFIX) + + +def get_workspace_info( + manifest_path: str | None = None, +) -> tuple[list[str], list[tuple[str, str]], list[tuple[str, str]]]: + """Return (all member names, normal edges, dev edges) for intra-workspace deps.""" + cmd = ["cargo", "metadata", "--format-version=1", "--no-deps"] + if manifest_path: + cmd += ["--manifest-path", manifest_path] + raw = subprocess.check_output(cmd, text=True) + meta = json.loads(raw) + members = {p["name"] for p in meta["packages"]} + + all_members = sorted(short(name) for name in members) + + normal_edges: list[tuple[str, str]] = [] + dev_edges: list[tuple[str, str]] = [] + for pkg in meta["packages"]: + if pkg["name"] not in members: + continue + for dep in pkg["dependencies"]: + if dep["name"] not in members: + continue + # Skip optional (feature-gated) deps + if dep.get("optional", False): + continue + edge = (short(pkg["name"]), short(dep["name"])) + if dep.get("kind") == "dev": + dev_edges.append(edge) + elif dep.get("kind") in (None, "normal"): + normal_edges.append(edge) + + return all_members, sorted(set(normal_edges)), sorted(set(dev_edges)) + + +def transitive_reduction( + edges: list[tuple[str, str]], +) -> list[tuple[str, str]]: + """Compute the transitive reduction (Hasse diagram) of a DAG.""" + # Build adjacency: src -> set of direct successors (dependencies). + adj: dict[str, set[str]] = {} + for src, dst in edges: + adj.setdefault(src, set()).add(dst) + adj.setdefault(dst, set()) + + # For each node, compute the full set of reachable nodes. + reachable: dict[str, set[str]] = {} + + def reach(n: str) -> set[str]: + if n in reachable: + return reachable[n] + r: set[str] = set() + for child in adj[n]: + r.add(child) + r |= reach(child) + reachable[n] = r + return r + + for n in adj: + reach(n) + + # An edge src→dst is redundant if dst is reachable from src + # through some other direct successor. + reduced: list[tuple[str, str]] = [] + for src, dst in edges: + others = adj[src] - {dst} + reachable_without = set() + for o in others: + reachable_without.add(o) + reachable_without |= reachable[o] + if dst not in reachable_without: + reduced.append((src, dst)) + + return sorted(reduced) + + +def topo_order(edges: list[tuple[str, str]]) -> dict[str, int]: + """Return a topological rank for each node (0 = leaf dependency).""" + nodes: set[str] = set() + for src, dst in edges: + nodes.add(src) + nodes.add(dst) + + children: dict[str, set[str]] = {n: set() for n in nodes} + in_degree: dict[str, int] = {n: 0 for n in nodes} + for src, dst in edges: + children[dst].add(src) + in_degree[src] += 1 + + order: dict[str, int] = {} + queue = sorted(n for n in nodes if in_degree[n] == 0) + rank = 0 + while queue: + next_queue: list[str] = [] + for n in queue: + order[n] = rank + for n in queue: + for child in children[n]: + in_degree[child] -= 1 + if in_degree[child] == 0: + next_queue.append(child) + queue = sorted(next_queue) + rank += 1 + + return order + + +def generate_mermaid( + all_members: list[str], + edges: list[tuple[str, str]], + dev_edges: list[tuple[str, str]] | None = None, + title: str | None = None, +) -> str: + all_edges = edges + (dev_edges or []) + nodes: set[str] = set(all_members) + for src, dst in all_edges: + nodes.add(src) + nodes.add(dst) + + order = topo_order(all_edges) + + # Group crates by prefix so Mermaid clusters them. + groups: dict[str, list[str]] = {} + grouped = {n for members in groups.values() for n in members} + + lines = ["```mermaid"] + if title: + lines.append("---") + lines.append(f"title: {title}") + lines.append("---") + lines.append("graph BT") + for label, members in groups.items(): + if members: + lines.append(f" subgraph {label}") + for n in members: + lines.append(f" {n}") + lines.append(" end") + + # Emit isolated nodes (no edges) that aren't in a subgraph. + connected = set() + for src, dst in all_edges: + connected.add(src) + connected.add(dst) + for n in sorted(nodes - connected - grouped): + lines.append(f" {n}") + + sorted_edges = sorted(edges, key=lambda e: (order.get(e[0], 0), e[0], e[1])) + for src, dst in sorted_edges: + lines.append(f" {src} --> {dst}") + + if dev_edges: + sorted_dev = sorted(dev_edges, key=lambda e: (order.get(e[0], 0), e[0], e[1])) + for src, dst in sorted_dev: + lines.append(f" {src} -.-> {dst}") + + lines.append("```") + return "\n".join(lines) + + +def main() -> None: + manifest_path = None + if "--manifest-path" in sys.argv: + idx = sys.argv.index("--manifest-path") + manifest_path = sys.argv[idx + 1] + + all_members, edges, dev_edges = get_workspace_info(manifest_path) + reduced = transitive_reduction(edges) + # For dev edges, reduce considering normal edges too (a dev edge is + # redundant if the target is already reachable via normal deps). + reduced_dev = transitive_reduction(edges + dev_edges) + reduced_dev = [e for e in reduced_dev if e not in set(reduced)] + mermaid = generate_mermaid(all_members, reduced, dev_edges=reduced_dev) + + # --doc PATH: specify the doc file (default: auto-detected from script location). + if "--doc" in sys.argv: + idx = sys.argv.index("--doc") + doc_file = Path(sys.argv[idx + 1]) + else: + doc_file = DEFAULT_DOC_FILE + + if doc_file.exists(): + text = doc_file.read_text() + blocks = list(re.finditer(r"```mermaid\n.*?```", text, flags=re.DOTALL)) + if blocks: + text = text[: blocks[0].start()] + mermaid + text[blocks[0].end() :] + else: + print("No mermaid block found in doc file", file=sys.stderr) + sys.exit(1) + + if "--update" in sys.argv: + doc_file.write_text(text) + print(f"Updated {doc_file}", file=sys.stderr) + else: + print(text, end="") + else: + print(mermaid) + + +if __name__ == "__main__": + main() diff --git a/sqlx-prepare.sh b/sqlx-prepare.sh new file mode 100755 index 000000000..d2ca0e97a --- /dev/null +++ b/sqlx-prepare.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +set -x + +PGDIR=$(mktemp -d) +trap 'pg_ctl -D "$PGDIR" stop -m immediate 2>/dev/null; rm -rf "$PGDIR"' EXIT + +initdb -D "$PGDIR" --no-locale -E UTF8 +pg_ctl -D "$PGDIR" -l "$PGDIR/log" -o "-k $PGDIR -h ''" start + +createdb -h "$PGDIR" hydra +psql -h "$PGDIR" -d hydra -f subprojects/hydra/sql/hydra.sql + +export DATABASE_URL="postgres://?host=$PGDIR&dbname=hydra" +#cd subprojects/crates/db +#ln -sfn ../../../.sqlx subprojects/crates/db/.sqlx +cargo sqlx prepare --workspace diff --git a/subprojects/crates/binary-cache/Cargo.toml b/subprojects/crates/binary-cache/Cargo.toml index 6ffcf3c0f..91efb8056 100644 --- a/subprojects/crates/binary-cache/Cargo.toml +++ b/subprojects/crates/binary-cache/Cargo.toml @@ -37,10 +37,19 @@ tokio = { workspace = true, features = [ "full" ] } tokio-stream = { workspace = true, features = [ "io-util" ] } tokio-util = { workspace = true, features = [ "io", "io-util" ] } -harmonia-store-core.workspace = true -harmonia-utils-hash.workspace = true -nix-utils = { path = "../nix-utils" } +daemon-client-utils.workspace = true +harmonia-file-core.workspace = true +harmonia-file-nar.workspace = true +harmonia-protocol.workspace = true +harmonia-store-derivation.workspace = true +harmonia-store-nar-info.workspace = true +harmonia-store-path.workspace = true +harmonia-store-path-info.workspace = true +harmonia-store-remote.workspace = true +harmonia-utils-hash.workspace = true +harmonia-utils-io.workspace = true +harmonia-utils-signature.workspace = true [dev-dependencies] -hydra-tracing = { path = "../tracing" } -tempfile = "3.23.0" +hydra-tracing.workspace = true +tempfile = "3.23.0" diff --git a/subprojects/crates/binary-cache/examples/download_file.rs b/subprojects/crates/binary-cache/examples/download_file.rs index e3e52c369..28c4c8f05 100644 --- a/subprojects/crates/binary-cache/examples/download_file.rs +++ b/subprojects/crates/binary-cache/examples/download_file.rs @@ -10,20 +10,24 @@ async fn main() -> Result<(), Box> { tracing::info!("{:#?}", client.cfg); let has_info = client - .has_narinfo(&nix_utils::parse_store_path( - "lmn7lwydprqibdkghw7wgcn21yhllz13-glibc-2.40-66", - )) + .has_narinfo( + &"lmn7lwydprqibdkghw7wgcn21yhllz13-glibc-2.40-66" + .parse::()?, + ) .await?; tracing::info!("has narinfo? {has_info}"); let narinfo = client - .download_narinfo(&nix_utils::parse_store_path( - "lmn7lwydprqibdkghw7wgcn21yhllz13-glibc-2.40-66", - )) + .download_narinfo( + &"lmn7lwydprqibdkghw7wgcn21yhllz13-glibc-2.40-66" + .parse::()?, + ) .await?; tracing::info!("narinfo:\n{narinfo:?}"); - let nardata = client.download_nar(&narinfo.unwrap().url).await?; + let nardata = client + .download_nar(narinfo.unwrap().info.url.as_deref().unwrap_or("")) + .await?; tracing::info!("nardata len: {}", nardata.unwrap().len()); let stats = client.s3_stats(); diff --git a/subprojects/crates/binary-cache/examples/query_missing_paths.rs b/subprojects/crates/binary-cache/examples/query_missing_paths.rs index 01d3660d5..58eceaa6d 100644 --- a/subprojects/crates/binary-cache/examples/query_missing_paths.rs +++ b/subprojects/crates/binary-cache/examples/query_missing_paths.rs @@ -1,10 +1,14 @@ use binary_cache::S3BinaryCacheClient; -use nix_utils::BaseStore as _; +use harmonia_store_path::StorePath; #[tokio::main] async fn main() -> Result<(), Box> { let _tracing_guard = hydra_tracing::init()?; - let store = nix_utils::LocalStore::init(); + let nix_config = daemon_client_utils::parse_nix_remote().unwrap(); + let store = harmonia_store_remote::ConnectionPool::new( + &nix_config.socket, + harmonia_store_remote::PoolConfig::default(), + ); let client = S3BinaryCacheClient::new( "s3://nix-cache-staging?ls-compression=br&log-compression=br".parse()?, @@ -12,16 +16,20 @@ async fn main() -> Result<(), Box> { .await?; tracing::info!("{:#?}", client.cfg); - let drv = nix_utils::parse_store_path("z3d15qi11dvljq5qz84kak3h0nb12wca-rsyslog-8.2510.0"); - let ps = store.query_requisites(&[&drv], false).await.unwrap(); - println!("ps before: {}", ps.len()); + let drv: StorePath = "z3d15qi11dvljq5qz84kak3h0nb12wca-rsyslog-8.2510.0".parse()?; + + let ps: Vec = daemon_client_utils::query_closure(&store, &[drv.clone()]) + .await? + .into_iter() + .map(|vpi| vpi.path) + .collect(); + println!("closure size: {}", ps.len()); - let ps = client.query_missing_paths(ps.clone()).await; - println!("ps after: {}", ps.len()); + let missing = client.query_missing_paths(ps).await; + println!("missing: {}", missing.len()); - let ps = store.query_requisites(&[&drv], true).await.unwrap(); - for p in ps { - println!("{}", store.print_store_path(&p)); + for p in &missing { + println!(" {}", store.store_dir().display(p)); } let stats = client.s3_stats(); diff --git a/subprojects/crates/binary-cache/examples/simple_presigned.rs b/subprojects/crates/binary-cache/examples/simple_presigned.rs index 7e7ba2c3a..22a59da62 100644 --- a/subprojects/crates/binary-cache/examples/simple_presigned.rs +++ b/subprojects/crates/binary-cache/examples/simple_presigned.rs @@ -1,15 +1,18 @@ use futures::stream::StreamExt as _; use binary_cache::{PresignedUploadClient, S3BinaryCacheClient, path_to_narinfo}; -use harmonia_utils_hash::fmt::CommonHash as _; -use nix_utils::BaseStore as _; +use harmonia_store_path::StorePath; #[tokio::main] async fn main() -> Result<(), Box> { let now = std::time::Instant::now(); let _tracing_guard = hydra_tracing::init()?; - let store = nix_utils::LocalStore::init(); + let nix_config = daemon_client_utils::parse_nix_remote().unwrap(); + let store = harmonia_store_remote::ConnectionPool::new( + &nix_config.socket, + harmonia_store_remote::PoolConfig::default(), + ); let client = S3BinaryCacheClient::new( format!( "s3://store2?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&write-debug-info=1&compression=zstd&ls-compression=br&log-compression=br&secret-key={}/../../example-secret-key&profile=local_nix_store", @@ -20,29 +23,24 @@ async fn main() -> Result<(), Box> { tracing::info!("{:#?}", client.cfg); let upload_client = PresignedUploadClient::new(); - let paths_to_copy = store - .query_requisites( - &[&nix_utils::parse_store_path( - "/nix/store/m1r53pnnm6hnjwyjmxska24y8amvlpjp-hello-2.12.1", - )], - true, - ) - .await - .unwrap_or_default(); + let root: StorePath = "m1r53pnnm6hnjwyjmxska24y8amvlpjp-hello-2.12.1".parse()?; + let paths_to_copy = daemon_client_utils::query_closure(&store, &[root]).await?; let mut stream = tokio_stream::iter(paths_to_copy) - .map(|p| { + .map(|vpi| { let client = client.clone(); let upload_client = upload_client.clone(); let store = store.clone(); + let p = vpi.path; async move { let narinfo = path_to_narinfo(&store, &p).await?; let presigned_request = client .generate_nar_upload_presigned_url( - &narinfo.store_path, - &format!("{}", narinfo.nar_hash.as_base32()), - binary_cache::get_debug_info_build_ids(&store, &p).await?, + &narinfo.path, + &narinfo.info.info.nar_hash, + binary_cache::get_debug_info_build_ids(store.store_dir().as_ref(), &p) + .await?, ) .await?; @@ -51,7 +49,7 @@ async fn main() -> Result<(), Box> { .await?; client - .upload_narinfo_after_presigned_upload(&store, narinfo) + .upload_narinfo_after_presigned_upload(narinfo) .await?; Ok::<(), Box>(()) } diff --git a/subprojects/crates/binary-cache/examples/upload_file.rs b/subprojects/crates/binary-cache/examples/upload_file.rs index 68a522d05..6f5dc3801 100644 --- a/subprojects/crates/binary-cache/examples/upload_file.rs +++ b/subprojects/crates/binary-cache/examples/upload_file.rs @@ -1,12 +1,16 @@ use binary_cache::S3BinaryCacheClient; -use nix_utils::BaseStore as _; +use harmonia_store_path::StorePath; #[tokio::main] async fn main() -> Result<(), Box> { let now = std::time::Instant::now(); let _tracing_guard = hydra_tracing::init()?; - let local = nix_utils::LocalStore::init(); + let nix_config = daemon_client_utils::parse_nix_remote().unwrap(); + let local = harmonia_store_remote::ConnectionPool::new( + &nix_config.socket, + harmonia_store_remote::PoolConfig::default(), + ); let client = S3BinaryCacheClient::new( format!( "s3://store2?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&compression=zstd&ls-compression=br&log-compression=br&secret-key={}/../../example-secret-key&profile=local_nix_store", @@ -16,15 +20,8 @@ async fn main() -> Result<(), Box> { .await?; tracing::info!("{:#?}", client.cfg); - let paths_to_copy = local - .query_requisites( - &[&nix_utils::parse_store_path( - "m1r53pnnm6hnjwyjmxska24y8amvlpjp-hello-2.12.1", - )], - true, - ) - .await - .unwrap_or_default(); + let root: StorePath = "m1r53pnnm6hnjwyjmxska24y8amvlpjp-hello-2.12.1".parse()?; + let paths_to_copy = daemon_client_utils::query_closure(&local, &[root]).await?; client.copy_paths(&local, paths_to_copy, true).await?; diff --git a/subprojects/crates/binary-cache/examples/upload_realisation.rs b/subprojects/crates/binary-cache/examples/upload_realisation.rs index f1d3e7eec..14fd277c7 100644 --- a/subprojects/crates/binary-cache/examples/upload_realisation.rs +++ b/subprojects/crates/binary-cache/examples/upload_realisation.rs @@ -3,7 +3,11 @@ use binary_cache::S3BinaryCacheClient; #[tokio::main] async fn main() -> Result<(), Box> { let _tracing_guard = hydra_tracing::init()?; - let local = nix_utils::LocalStore::init(); + let nix_config = daemon_client_utils::parse_nix_remote().unwrap(); + let _local = harmonia_store_remote::ConnectionPool::new( + &nix_config.socket, + harmonia_store_remote::PoolConfig::default(), + ); let client = S3BinaryCacheClient::new( format!( "s3://store2?region=unknown&endpoint=http://localhost:9000&scheme=http&write-nar-listing=1&compression=zstd&ls-compression=br&log-compression=br&secret-key={}/../../example-secret-key&profile=local_nix_store", @@ -13,22 +17,25 @@ async fn main() -> Result<(), Box> { .await?; tracing::info!("{:#?}", client.cfg); - let id = nix_utils::DrvOutput { - drv_hash: "sha256:6e46b9cf4fecaeab4b3c0578f4ab99e89d2f93535878c4ac69b5d5c4eb3a3db9" - .parse::>() - .unwrap() - .into_hash(), + let id = harmonia_store_derivation::realisation::DrvOutput { + drv_path: "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bash-5.2p37.drv" + .parse() + .unwrap(), output_name: "debug".parse().unwrap(), }; tracing::info!( "has realisation before: {}", client.has_realisation(&id).await? ); - client.copy_realisation(&local, &id, true).await?; - tracing::info!( - "has realisation after: {}", - client.has_realisation(&id).await? - ); + // TODO put back after we add back `query_raw_realisation` with Nix 2.35. + + // let raw = local.query_raw_realisation(&id)?; + // let realisation = raw.as_rust()?; + // client.write_realisation(realisation).await?; + // tracing::info!( + // "has realisation after: {}", + // client.has_realisation(&id).await? + // ); let stats = client.s3_stats(); tracing::info!( diff --git a/subprojects/crates/binary-cache/src/cfg.rs b/subprojects/crates/binary-cache/src/cfg.rs index 454123969..716de77c7 100644 --- a/subprojects/crates/binary-cache/src/cfg.rs +++ b/subprojects/crates/binary-cache/src/cfg.rs @@ -1,3 +1,4 @@ +use harmonia_store_path::StoreDir; use hashbrown::HashMap; use smallvec::SmallVec; @@ -11,6 +12,10 @@ const MAX_PRESIGNED_URL_EXPIRY_SECS: u64 = 24 * 60 * 60; pub struct S3CacheConfig { pub client_config: S3ClientConfig, + /// Store directory the cache's narinfos refer to, from the `store` query + /// parameter of the nix S3 URL (defaults to the standard `/nix/store`). + pub store_dir: StoreDir, + pub compression: Compression, pub write_nar_listing: bool, pub write_debug_info: bool, @@ -32,6 +37,7 @@ impl S3CacheConfig { pub fn new(client_config: S3ClientConfig) -> Self { Self { client_config, + store_dir: StoreDir::default(), compression: Compression::Xz, write_nar_listing: false, write_debug_info: false, @@ -43,7 +49,7 @@ impl S3CacheConfig { ls_compression: Compression::None, log_compression: Compression::None, buffer_size: 8 * 1024 * 1024, - presigned_url_expiry: std::time::Duration::from_secs(3600), + presigned_url_expiry: std::time::Duration::from_hours(1), } } @@ -139,6 +145,14 @@ impl S3CacheConfig { self } + #[must_use] + pub fn with_store_dir(mut self, store_dir: Option) -> Self { + if let Some(store_dir) = store_dir { + self.store_dir = store_dir; + } + self + } + pub fn with_presigned_url_expiry( mut self, expiry_secs: Option, @@ -173,12 +187,14 @@ pub enum UrlParseError { UriParseError(#[from] url::ParseError), #[error("Int parse error: {0}")] IntParseError(#[from] std::num::ParseIntError), - #[error("Invalid S3Scheme: {0}")] - S3SchemeParseError(String), - #[error("Invalid Compression: {0}")] - CompressionParseError(String), - #[error("Bad schema: {0}")] - BadSchema(String), + #[error("Invalid store directory: {0}")] + StoreDir(String), + #[error(transparent)] + S3Scheme(#[from] InvalidS3Scheme), + #[error(transparent)] + Compression(#[from] crate::compression::InvalidCompression), + #[error("unsupported URI scheme: {0:?} (expected \"s3\")")] + UnsupportedScheme(String), #[error("Bucket not defined")] NoBucket, #[error("Invalid presigned URL expiry: {0}. Must be between {1} and {2} seconds")] @@ -192,7 +208,7 @@ impl std::str::FromStr for S3CacheConfig { fn from_str(s: &str) -> Result { let uri = url::Url::parse(&s.trim().to_ascii_lowercase())?; if uri.scheme() != "s3" { - return Err(UrlParseError::BadSchema(uri.scheme().to_owned())); + return Err(UrlParseError::UnsupportedScheme(uri.scheme().to_owned())); } let bucket = uri.authority(); if bucket.is_empty() { @@ -205,19 +221,24 @@ impl std::str::FromStr for S3CacheConfig { query .get("scheme") .map(|x| x.parse::()) - .transpose() - .map_err(UrlParseError::S3SchemeParseError)?, + .transpose()?, ) .with_endpoint(query.get("endpoint").map(String::as_str)) .with_profile(query.get("profile").map(String::as_str)); Self::new(cfg) + .with_store_dir( + query + .get("store") + .map(|x| x.parse::()) + .transpose() + .map_err(|e| UrlParseError::StoreDir(e.to_string()))?, + ) .with_compression( query .get("compression") .map(|x| x.parse::()) - .transpose() - .map_err(UrlParseError::CompressionParseError)?, + .transpose()?, ) .with_write_nar_listing(query.get("write-nar-listing").map(String::as_str)) .with_write_debug_info(query.get("write-debug-info").map(String::as_str)) @@ -249,22 +270,19 @@ impl std::str::FromStr for S3CacheConfig { query .get("narinfo-compression") .map(|x| x.parse::()) - .transpose() - .map_err(UrlParseError::CompressionParseError)?, + .transpose()?, ) .with_ls_compression( query .get("ls-compression") .map(|x| x.parse::()) - .transpose() - .map_err(UrlParseError::CompressionParseError)?, + .transpose()?, ) .with_log_compression( query .get("log-compression") .map(|x| x.parse::()) - .transpose() - .map_err(UrlParseError::CompressionParseError)?, + .transpose()?, ) .with_buffer_size( query @@ -288,14 +306,21 @@ pub enum S3Scheme { HTTPS, } +/// Invalid S3 URI scheme (expected `http` or `https`). +#[derive(Debug, Clone, thiserror::Error)] +#[error("invalid S3 scheme: {got:?} (expected \"http\" or \"https\")")] +pub struct InvalidS3Scheme { + pub got: String, +} + impl std::str::FromStr for S3Scheme { - type Err = String; + type Err = InvalidS3Scheme; fn from_str(s: &str) -> Result { match s.trim().to_ascii_lowercase().as_str() { "http" => Ok(Self::HTTP), "https" => Ok(Self::HTTPS), - v => Err(v.to_owned()), + v => Err(InvalidS3Scheme { got: v.to_owned() }), } } } @@ -445,7 +470,7 @@ mod tests { use super::{ Compression, ConfigReadError, S3CacheConfig, S3ClientConfig, S3CredentialsConfig, S3Scheme, - UrlParseError, parse_aws_credentials_file, + StoreDir, UrlParseError, parse_aws_credentials_file, }; use std::str::FromStr as _; @@ -808,9 +833,20 @@ aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvb123KEY" assert!(config.client_config.profile.is_none()); } + #[test] + fn test_s3_cache_config_from_str_store_dir() { + // Defaults to the standard store directory when `store` is absent. + let config = S3CacheConfig::from_str("s3://my-bucket").unwrap(); + assert_eq!(config.store_dir, StoreDir::default()); + + // The `store` query param overrides it, matching nix S3 store URLs. + let config = S3CacheConfig::from_str("s3://my-bucket?store=/custom/store").unwrap(); + assert_eq!(config.store_dir.to_string(), "/custom/store"); + } + #[test] fn test_s3_cache_config_from_str_with_parameters() { - let config_str = "s3://test-bucket?region=eu-west-1&scheme=http&endpoint=custom.example.com&profile=myprofile&compression=zstd&write-nar-listing=true&write-debug-info=1¶llel-compression=true&compression-level=9&narinfo-compression=bz2&ls-compression=br&log-compression=xz&buffer-size=16777216&presigned-url-expiry=7200"; + let config_str = "s3://test-bucket?region=eu-west-1&scheme=http&endpoint=custom.example.com&profile=myprofile&compression=zstd&write-nar-listing=true&write-debug-info=1¶llel-compression=true&compression-level=9&narinfo-compression=bzip2&ls-compression=br&log-compression=xz&buffer-size=16777216&presigned-url-expiry=7200"; let config = S3CacheConfig::from_str(config_str).unwrap(); @@ -893,7 +929,7 @@ aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvb123KEY" assert!(!config.write_debug_info); assert!(config.parallel_compression); - let config_str = "s3://test-bucket?compression=XZ&narinfo-compression=BZ2&ls-compression=BR&log-compression=ZSTD"; + let config_str = "s3://test-bucket?compression=XZ&narinfo-compression=BZIP2&ls-compression=BR&log-compression=ZSTD"; let config = S3CacheConfig::from_str(config_str).unwrap(); assert_eq!(config.compression, Compression::Xz); assert_eq!(config.narinfo_compression, Compression::Bzip2); @@ -905,7 +941,10 @@ aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvb123KEY" fn test_s3_cache_config_from_str_errors() { let result = S3CacheConfig::from_str("http://test-bucket"); assert!(result.is_err()); - assert!(matches!(result.unwrap_err(), UrlParseError::BadSchema(_))); + assert!(matches!( + result.unwrap_err(), + UrlParseError::UnsupportedScheme(_) + )); let result = S3CacheConfig::from_str("s3://"); assert!(result.is_err()); @@ -913,17 +952,11 @@ aws_secret_access_key = je7MtGbClwBF/2Zp9Utk/h3yCo8nvb123KEY" let result = S3CacheConfig::from_str("s3://test-bucket?compression=invalid"); assert!(result.is_err()); - assert!(matches!( - result.unwrap_err(), - UrlParseError::CompressionParseError(_) - )); + assert!(matches!(result.unwrap_err(), UrlParseError::Compression(_))); let result = S3CacheConfig::from_str("s3://test-bucket?scheme=invalid"); assert!(result.is_err()); - assert!(matches!( - result.unwrap_err(), - UrlParseError::S3SchemeParseError(_) - )); + assert!(matches!(result.unwrap_err(), UrlParseError::S3Scheme(_))); let result = S3CacheConfig::from_str("s3://test-bucket?compression-level=invalid"); assert!(result.is_err()); diff --git a/subprojects/crates/binary-cache/src/compression.rs b/subprojects/crates/binary-cache/src/compression.rs index 2d113334b..eaef747bb 100644 --- a/subprojects/crates/binary-cache/src/compression.rs +++ b/subprojects/crates/binary-cache/src/compression.rs @@ -42,7 +42,7 @@ impl Compression { match self { Self::None => "none", Self::Xz => "xz", - Self::Bzip2 => "bz2", + Self::Bzip2 => "bzip2", Self::Brotli => "br", Self::Zstd => "zstd", } @@ -70,17 +70,24 @@ impl Compression { } } +/// Invalid compression type string. +#[derive(Debug, Clone, thiserror::Error)] +#[error("invalid compression: {got:?} (expected \"none\", \"xz\", \"bzip2\", \"br\", or \"zstd\")")] +pub struct InvalidCompression { + pub got: String, +} + impl std::str::FromStr for Compression { - type Err = String; + type Err = InvalidCompression; fn from_str(s: &str) -> Result { match s.trim().to_ascii_lowercase().as_str() { "none" => Ok(Self::None), "xz" => Ok(Self::Xz), - "bz2" => Ok(Self::Bzip2), + "bzip2" => Ok(Self::Bzip2), "br" => Ok(Self::Brotli), "zstd" | "zst" => Ok(Self::Zstd), - o => Err(o.to_string()), + o => Err(InvalidCompression { got: o.to_string() }), } } } diff --git a/subprojects/crates/binary-cache/src/debug_info.rs b/subprojects/crates/binary-cache/src/debug_info.rs index 0f673604a..4e616dcee 100644 --- a/subprojects/crates/binary-cache/src/debug_info.rs +++ b/subprojects/crates/binary-cache/src/debug_info.rs @@ -4,8 +4,6 @@ //! from NIX store paths that contain debug symbols in the standard //! `lib/debug/.build-id` directory structure. -use nix_utils::BaseStore as _; - use crate::CacheError; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -18,8 +16,8 @@ pub(crate) struct DebugInfoLink { /// This is useful for testing with custom store prefixes. pub(crate) async fn process_debug_info( nar_url: &str, - store: &nix_utils::LocalStore, - store_path: &nix_utils::StorePath, + real_store_dir: &std::path::Path, + store_path: &harmonia_store_path::StorePath, client: C, ) -> Result<(), CacheError> where @@ -27,8 +25,8 @@ where { use futures::stream::StreamExt as _; - let full_path = store.print_store_path(store_path); - let build_id_path = std::path::Path::new(&full_path).join("lib/debug/.build-id"); + let full_path = real_store_dir.join(store_path.to_string()); + let build_id_path = full_path.join("lib/debug/.build-id"); if !build_id_path.exists() { tracing::debug!("No lib/debug/.build-id directory found in {}", store_path); @@ -53,11 +51,11 @@ where } pub async fn get_debug_info_build_ids( - store: &nix_utils::LocalStore, - store_path: &nix_utils::StorePath, + real_store_dir: &std::path::Path, + store_path: &harmonia_store_path::StorePath, ) -> Result, CacheError> { - let full_path = store.print_store_path(store_path); - let build_id_path = std::path::Path::new(&full_path).join("lib/debug/.build-id"); + let full_path = real_store_dir.join(store_path.to_string()); + let build_id_path = full_path.join("lib/debug/.build-id"); if !build_id_path.exists() { tracing::debug!("No lib/debug/.build-id directory found in {}", store_path); @@ -72,6 +70,7 @@ pub async fn get_debug_info_build_ids( } /// Finds debug files by scanning the build-id directory structure. +#[allow(clippy::case_sensitive_file_extension_comparisons)] async fn find_debug_files( build_id_path: &std::path::Path, ) -> Result, CacheError> { @@ -81,37 +80,98 @@ async fn find_debug_files( .await .map_err(CacheError::Io)?; + // Debuginfo is always stored in a directory called {aa}/{bb...}.debug, + // where {aa} and {bb...} are hexadecimal. + // gdb and elfutils assume that the hexadecimal and "debug" are lowercase. + // The concatenation of {aa} and {bb...} represent the ID in the ELF's + // `.note.gnu.build-id` section, and must be a whole number of bytes. + // GNU ld, gold, mold, and lld are capable of using a user-specified ID or + // an automatically-generated ID of 8, 16, or 20 bytes. + // elfutils assumes all build IDs are 3–64 bytes (inclusive), + + // The elfutils and gdb assumptions are reasonable, so we can limit ourselves + // to 3–64 bytes worth of lowercase hexadecimal followed by a lowercase ".debug" + while let Some(entry) = entries.next_entry().await.map_err(CacheError::Io)? { - let s1 = entry.file_name(); - let s1_str = s1.to_string_lossy(); + let Ok(outer_name) = entry.file_name().into_string() else { + tracing::warn!( + "Skipping build-id entry with a non-UTF-8 name: {}", + entry.path().display() + ); + continue; + }; // Check if it's a 2-character hex directory - if s1_str.len() != 2 - || !s1_str.chars().all(|c| c.is_ascii_hexdigit()) + if outer_name.len() != 2 + || !outer_name + .chars() + .all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)) || !entry.file_type().await.map_err(CacheError::Io)?.is_dir() { + tracing::debug!( + "Skipping unexpected entry in .build-id: {}", + entry.path().display() + ); continue; } - let subdir_path = build_id_path.join(&s1); + let subdir_path = build_id_path.join(&outer_name); let mut subdir_entries = fs_err::tokio::read_dir(&subdir_path) .await .map_err(CacheError::Io)?; while let Some(sub_entry) = subdir_entries.next_entry().await.map_err(CacheError::Io)? { - let s2 = sub_entry.file_name(); - let s2_str = s2.to_string_lossy(); - - // Check if it's a 38-character hex file ending with .debug - if s2_str.len() == 44 // 38 chars + .debug (6 chars) = 44 - && s2_str.ends_with(".debug") - && s2_str[..38].chars().all(|c| c.is_ascii_hexdigit()) - && sub_entry.file_type().await.map_err(CacheError::Io)?.is_file() + let sub_path = sub_entry.path(); + if sub_path.extension() != Some("debug".as_ref()) { + tracing::debug!("Skipping non-debug file: {}", sub_path.display()); + continue; + } + + // `file_stem` only fails to produce a `&str` for a non-UTF-8 name, + // which a real build ID should never have. + let Some(sub_name) = sub_path.file_stem().and_then(|name| name.to_str()) else { + tracing::warn!( + "Skipping debug file with a non-UTF-8 name: {}", + sub_path.display() + ); + continue; + }; + + // The build ID format is `{outer_name}{sub_name}` in hex, and takes two chars per byte. + let build_id_hex_chars = outer_name.len() + sub_name.len(); + let build_id_bytes = build_id_hex_chars / 2; + if !(3..=64).contains(&build_id_bytes) { + tracing::debug!( + "Skipping debug file with a build ID of {build_id_bytes} bytes, expected 3-64: {}", + sub_path.display() + ); + continue; + } + + if !sub_name + .chars() + .all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)) + { + tracing::debug!( + "Skipping debug file with a non-hexadecimal build ID: {}", + sub_path.display() + ); + continue; + } + + if !sub_entry + .file_type() + .await + .map_err(CacheError::Io)? + .is_file() { - let build_id = format!("{s1_str}{}", &s2_str[..38]); - let debug_path = format!("lib/debug/.build-id/{s1_str}/{s2_str}"); - debug_files.push((build_id, debug_path)); + tracing::debug!("Skipping non-file debug entry: {}", sub_path.display()); + continue; } + + let build_id = format!("{outer_name}{sub_name}"); + let debug_path = format!("lib/debug/.build-id/{outer_name}/{sub_name}.debug"); + debug_files.push((build_id, debug_path)); } } @@ -221,7 +281,8 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap().keep(); let store_prefix = temp_dir.join("nix/store"); let store_path_str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-debug-output"; - let store_path = nix_utils::parse_store_path(store_path_str); + let store_dir = harmonia_store_path::StoreDir::new(store_prefix.as_path()).unwrap(); + let store_path = harmonia_store_path::StorePath::from_base_path(store_path_str).unwrap(); let full_path = store_prefix.join(store_path_str); fs_err::tokio::create_dir_all(&full_path).await.unwrap(); @@ -235,12 +296,14 @@ mod tests { .await .unwrap(); - let mut local = nix_utils::LocalStore::init(); - local.unsafe_set_store_dir(nix_utils::StoreDir::new(store_prefix.as_path()).unwrap()); - - process_debug_info("test.nar", &local, &store_path, mock_client.clone()) - .await - .unwrap(); + process_debug_info( + "test.nar", + store_dir.as_ref(), + &store_path, + mock_client.clone(), + ) + .await + .unwrap(); let created_links = mock_client.get_created_links(); assert_eq!(created_links.len(), 1); @@ -289,15 +352,6 @@ mod tests { fs_err::tokio::write(&valid_dir.join("invalid.txt"), "content") .await .unwrap(); - fs_err::tokio::write(&valid_dir.join("cdef.debug"), "content") - .await - .unwrap(); - fs_err::tokio::write( - &valid_dir.join("cdef12345678901234567890123456789012345.debug"), - "content", - ) - .await - .unwrap(); fs_err::tokio::write( &valid_dir.join("cdef1234567890123456789012345678901234.txt"), "content", @@ -327,7 +381,7 @@ mod tests { let ab_dir = build_id_dir.join("ab"); fs_err::tokio::create_dir(&ab_dir).await.unwrap(); fs_err::tokio::write( - &ab_dir.join("cdef1234567890123456789012345678901234.debug"), + &ab_dir.join("cdef123456789012345678901234567890123456.debug"), "valid debug content", ) .await @@ -359,7 +413,7 @@ mod tests { assert_eq!(debug_files.len(), 2); let build_ids: Vec = debug_files.iter().map(|(id, _)| id.clone()).collect(); - assert!(build_ids.contains(&"abcdef1234567890123456789012345678901234".to_string())); + assert!(build_ids.contains(&"abcdef123456789012345678901234567890123456".to_string())); assert!(build_ids.contains(&"cdef567890123456789012345678901234567890".to_string())); fs_err::tokio::remove_dir_all(&build_id_dir).await.unwrap(); @@ -372,17 +426,20 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap().keep(); let store_prefix = temp_dir.join("nix/store"); let store_path_str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-no-debug"; - let store_path = nix_utils::parse_store_path(store_path_str); + let store_dir = harmonia_store_path::StoreDir::new(store_prefix.as_path()).unwrap(); + let store_path = harmonia_store_path::StorePath::from_base_path(store_path_str).unwrap(); let full_path = temp_dir.join("nix/store").join(store_path_str); fs_err::tokio::create_dir_all(&full_path).await.unwrap(); - let mut local = nix_utils::LocalStore::init(); - local.unsafe_set_store_dir(nix_utils::StoreDir::new(store_prefix.as_path()).unwrap()); - - process_debug_info("test.nar", &local, &store_path, mock_client.clone()) - .await - .unwrap(); + process_debug_info( + "test.nar", + store_dir.as_ref(), + &store_path, + mock_client.clone(), + ) + .await + .unwrap(); let created_links = mock_client.get_created_links(); assert_eq!(created_links.len(), 0); @@ -397,19 +454,22 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap().keep(); let store_prefix = temp_dir.join("nix/store"); let store_path_str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-empty-debug"; - let store_path = nix_utils::parse_store_path(store_path_str); + let store_dir = harmonia_store_path::StoreDir::new(store_prefix.as_path()).unwrap(); + let store_path = harmonia_store_path::StorePath::from_base_path(store_path_str).unwrap(); let full_path = temp_dir.join("nix/store").join(store_path_str); fs_err::tokio::create_dir_all(&full_path).await.unwrap(); let build_id_dir = full_path.join("lib/debug/.build-id"); fs_err::tokio::create_dir_all(&build_id_dir).await.unwrap(); - let mut local = nix_utils::LocalStore::init(); - local.unsafe_set_store_dir(nix_utils::StoreDir::new(store_prefix.as_path()).unwrap()); - - process_debug_info("test.nar", &local, &store_path, mock_client.clone()) - .await - .unwrap(); + process_debug_info( + "test.nar", + store_dir.as_ref(), + &store_path, + mock_client.clone(), + ) + .await + .unwrap(); let created_links = mock_client.get_created_links(); assert_eq!(created_links.len(), 0); @@ -424,7 +484,8 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap().keep(); let store_prefix = temp_dir.join("nix/store"); let store_path_str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-multi-debug"; - let store_path = nix_utils::parse_store_path(store_path_str); + let store_dir = harmonia_store_path::StoreDir::new(store_prefix.as_path()).unwrap(); + let store_path = harmonia_store_path::StorePath::from_base_path(store_path_str).unwrap(); let full_path = temp_dir.join("nix/store").join(store_path_str); fs_err::tokio::create_dir_all(&full_path).await.unwrap(); @@ -445,12 +506,14 @@ mod tests { .unwrap(); } - let mut local = nix_utils::LocalStore::init(); - local.unsafe_set_store_dir(nix_utils::StoreDir::new(store_prefix.as_path()).unwrap()); - - process_debug_info("multi.nar", &local, &store_path, mock_client.clone()) - .await - .unwrap(); + process_debug_info( + "multi.nar", + store_dir.as_ref(), + &store_path, + mock_client.clone(), + ) + .await + .unwrap(); let created_links = mock_client.get_created_links(); assert_eq!(created_links.len(), 3); diff --git a/subprojects/crates/binary-cache/src/lib.rs b/subprojects/crates/binary-cache/src/lib.rs index e63e33c01..4023884ce 100644 --- a/subprojects/crates/binary-cache/src/lib.rs +++ b/subprojects/crates/binary-cache/src/lib.rs @@ -24,7 +24,8 @@ use object_store::{ObjectStore as _, ObjectStoreExt as _, signer::Signer as _}; use secrecy::ExposeSecret; use smallvec::SmallVec; -use nix_utils::BaseStore as _; +use harmonia_store_path::StorePath; + // Realisation writing is now done by the caller, not via FFI query. mod cfg; @@ -37,8 +38,10 @@ mod streaming_hash; pub use crate::cfg::{S3CacheConfig, S3ClientConfig, S3CredentialsConfig, S3Scheme}; pub use crate::compression::Compression; pub use crate::debug_info::get_debug_info_build_ids; -use crate::narinfo::NarInfoError; -pub use crate::narinfo::{NarInfo, parse_hash}; +pub use crate::narinfo::{ + NarInfo, clear_sigs_and_sign, format_narinfo_txt, get_ls_path, narinfo_from_path_info, + narinfo_simple, parse_hash, parse_nar_hash, parse_narinfo_txt, +}; pub use crate::presigned::{ PresignedUpload, PresignedUploadClient, PresignedUploadMetrics, PresignedUploadResponse, PresignedUploadResult, @@ -47,24 +50,21 @@ pub use async_compression::Level as CompressionLevel; pub use harmonia_utils_hash::{self as harmonia_utils_hash, Hash}; pub async fn path_to_narinfo( - store: &nix_utils::LocalStore, - path: &nix_utils::StorePath, + store: &harmonia_store_remote::ConnectionPool, + path: &StorePath, ) -> Result { - let Some(path_info) = store.query_path_info(path).await else { - return Err(CacheError::PathNotFound { + let path_info = daemon_client_utils::query_path_info(store, path) + .await? + .ok_or_else(|| CacheError::PathNotFound { path: path.to_string(), - }); - }; - let narinfo = NarInfo::simple(path, path_info, Compression::None); - let queried_references = store - .query_path_infos(&narinfo.references.iter().collect::>()) - .await; - for r in &narinfo.references { - if !queried_references.contains_key(r) { - return Err(CacheError::ReferenceVerifyError( - narinfo.store_path, - r.to_owned(), - )); + })?; + let narinfo = narinfo_simple(path, path_info, Compression::None); + for r in &narinfo.info.info.references { + if daemon_client_utils::query_path_info(store, r) + .await? + .is_none() + { + return Err(CacheError::ReferenceVerifyError(narinfo.path, r.to_owned())); } } Ok(narinfo) @@ -124,6 +124,24 @@ impl S3Stats { } } +#[derive(Debug, thiserror::Error)] +pub enum NarReadErrorInner { + #[error("Connecting to Nix daemon")] + DaemonPool(#[source] harmonia_store_remote::DaemonError), + #[error("Requesting nar from Nix daemon")] + RequestingNar(#[source] harmonia_store_remote::DaemonError), + #[error("Reading nar from Nix daemon")] + Reading(#[source] std::io::Error), +} + +#[derive(Debug, thiserror::Error)] +#[error("Failed to serialise nar from store path {path}")] +pub struct NarReadError { + #[source] + source: NarReadErrorInner, + path: StorePath, +} + #[derive(Debug, thiserror::Error)] pub enum CacheError { #[error("Object store error: {0}")] @@ -134,20 +152,26 @@ pub enum CacheError { Serde(#[from] serde_json::Error), #[error("Signing error: {0}")] Signing(String), + #[error("narinfo was not valid utf-8")] + NarInfoUtf8(#[from] std::str::Utf8Error), #[error(transparent)] - NarInfoParseError(#[from] NarInfoError), - #[error(transparent)] - NixStoreError(#[from] nix_utils::Error), + NarInfoParseError(#[from] harmonia_store_nar_info::NarInfoParseError), + #[error("daemon error: {0}")] + DaemonError(#[from] harmonia_protocol::types::DaemonError), #[error("cannot add '{0}' to the binary cache because the reference '{1}' is not valid")] - ReferenceVerifyError(nix_utils::StorePath, nix_utils::StorePath), + ReferenceVerifyError(StorePath, StorePath), #[error("Hash error: {0}")] HashingError(#[from] streaming_hash::Error), #[error("Render error: {0}")] RenderError(#[from] std::fmt::Error), #[error("HTTP request failed: {0}")] HttpRequestError(#[from] reqwest::Error), - #[error("Upload failed for {path}: {reason}")] - UploadError { path: String, reason: String }, + #[error("Upload failed for {path}")] + Upload { + path: String, + #[source] + source: Box, + }, #[error("Presigned URL generation failed for {path}: {reason}")] PresignedUrlError { path: String, reason: String }, #[error("Request cloning failed")] @@ -161,10 +185,10 @@ pub enum CacheError { #[derive(Debug, Clone)] pub struct S3BinaryCacheClient { s3: object_store::aws::AmazonS3, - pub cfg: S3CacheConfig, + pub cfg: Arc, s3_stats: Arc, signing_keys: SmallVec<[secrecy::SecretString; 4]>, - narinfo_cache: Cache, + narinfo_cache: Cache, } #[tracing::instrument(skip(stream, chunk), err)] @@ -225,6 +249,62 @@ async fn run_multipart_upload( Ok(file_size) } +fn read_nar_stream( + store: &harmonia_store_remote::ConnectionPool, + path: &StorePath, +) -> tokio_util::io::StreamReader< + tokio_stream::wrappers::UnboundedReceiverStream>, + Bytes, +> { + // Has to be std::io::Error because of StreamReader, but it will still pass through source + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); + + tokio::task::spawn({ + let path = path.clone(); + let store = store.clone(); + async move { + use harmonia_protocol::types::DaemonStore; + use tokio::io::AsyncBufReadExt as _; + let result: Result<(), NarReadErrorInner> = async { + let mut guard = store + .acquire() + .await + .map_err(NarReadErrorInner::DaemonPool)?; + let mut nar_reader = guard + .client() + .nar_from_path(&path) + .await + .map_err(NarReadErrorInner::RequestingNar)?; + loop { + let buf = nar_reader + .fill_buf() + .await + .map_err(NarReadErrorInner::Reading)?; + if buf.is_empty() { + break; + } + let data = Bytes::copy_from_slice(buf); + let len = data.len(); + if tx.send(Ok(data)).is_err() { + break; + } + nar_reader.consume(len); + } + Ok(()) + } + .await; + if let Err(source) = result { + let _ = tx.send(Err(std::io::Error::other(NarReadError { + source, + path: path.clone(), + }))); + } + } + }); + + tokio_util::io::StreamReader::new(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)) +} + impl S3BinaryCacheClient { #[tracing::instrument(skip(cfg), err)] fn construct_client( @@ -283,7 +363,7 @@ impl S3BinaryCacheClient { Ok(Self { s3: Self::construct_client(&cfg.client_config)?, - cfg, + cfg: cfg.into(), s3_stats: Arc::new(AtomicS3Stats::default()), signing_keys, narinfo_cache: Cache::builder() @@ -341,13 +421,13 @@ impl S3BinaryCacheClient { } #[tracing::instrument(skip(self, content, content_type), err)] - pub async fn upsert_file( + pub async fn upsert_file>( &self, name: &str, - content: String, + content: T, content_type: &str, ) -> Result<(), CacheError> { - let stream = Box::new(std::io::Cursor::new(Bytes::from(content))); + let stream = Box::new(std::io::Cursor::new(content.into())); self.upsert_file_stream(name, stream, content_type).await } @@ -523,101 +603,81 @@ impl S3BinaryCacheClient { Ok(()) } - #[tracing::instrument(skip(self, store_dir, narinfo), err)] - async fn upload_narinfo( - &self, - store_dir: &nix_utils::StoreDir, - narinfo: NarInfo, - ) -> Result { - let base = narinfo.store_path.hash().to_string(); + #[tracing::instrument(skip(self, narinfo), err)] + async fn upload_narinfo(&self, narinfo: NarInfo) -> Result { + let base = narinfo.path.hash().to_string(); let info_key = format!("{base}.narinfo"); - self.upsert_file(&info_key, narinfo.render(store_dir)?, "text/x-nix-narinfo") - .await?; + self.upsert_file( + &info_key, + format_narinfo_txt(&self.cfg.store_dir, &narinfo), + "text/x-nix-narinfo", + ) + .await?; Ok(info_key) } - #[tracing::instrument(skip(self, store), fields(%path), err)] - async fn path_to_narinfo( + fn narinfo_from_valid_path_info( &self, - store: &nix_utils::LocalStore, - path: &nix_utils::StorePath, - ) -> Result { - let Some(path_info) = store.query_path_info(path).await else { - return Err(CacheError::PathNotFound { - path: path.to_string(), - }); - }; - let narinfo = NarInfo::new( - path, - path_info, + vpi: &harmonia_store_path_info::ValidPathInfo, + ) -> NarInfo { + narinfo_from_path_info( + &vpi.path, + vpi.info.clone(), self.cfg.compression, - store.get_store_dir(), + &self.cfg.store_dir, &self.signing_keys, - ); - let queried_references = store - .query_path_infos(&narinfo.references.iter().collect::>()) - .await; - for r in &narinfo.references { - if !queried_references.contains_key(r) { - return Err(CacheError::ReferenceVerifyError( - narinfo.store_path, - r.to_owned(), - )); - } - } - Ok(narinfo) + ) } - #[tracing::instrument(skip(self, store), err)] + #[tracing::instrument(skip(self, store, vpi), fields(%vpi.path), err)] pub async fn copy_path( &self, - store: &nix_utils::LocalStore, - path: &nix_utils::StorePath, + store: &harmonia_store_remote::ConnectionPool, + vpi: &harmonia_store_path_info::ValidPathInfo, repair: bool, ) -> Result<(), CacheError> { - if !repair && self.has_narinfo(path).await? { + if !repair && self.has_narinfo(&vpi.path).await? { return Ok(()); } - tracing::debug!("start copying path: {path}"); - let mut narinfo = self.path_to_narinfo(store, path).await?; + tracing::debug!("start copying path: {}", vpi.path); + let mut narinfo = self.narinfo_from_valid_path_info(vpi); if self.cfg.write_nar_listing { - let ls = store.list_nar_deep(&narinfo.store_path).await?; - self.upload_listing(&narinfo.get_ls_path(), ls).await?; + let listing = nar_listing(store, &narinfo.path).await?; + let ls_json = serde_json::json!({ + "version": 1, + "root": listing, + }); + self.upload_listing(&get_ls_path(&narinfo), ls_json.to_string()) + .await?; } + let nar_url = narinfo.info.url.clone().unwrap_or_default(); + let compression = self.cfg.compression; + if self.cfg.write_debug_info { - debug_info::process_debug_info(&narinfo.url, store, &narinfo.store_path, self.clone()) - .await?; + debug_info::process_debug_info( + &nar_url, + self.cfg.store_dir.as_ref(), + &narinfo.path, + self.clone(), + ) + .await?; } - let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); - let closure = move |data: &[u8]| { - let data = Bytes::copy_from_slice(data); - tx.send(Ok(data)).is_ok() - }; + let stream = read_nar_stream(store, &narinfo.path); - tokio::task::spawn({ - let path = narinfo.store_path.clone(); - let store = store.clone(); - async move { - let _ = store.nar_from_path(&path, closure); - } - }); - let stream = tokio_util::io::StreamReader::new( - tokio_stream::wrappers::UnboundedReceiverStream::new(rx), - ); - let compressor = narinfo.compression.get_compression_fn( + let compressor = compression.get_compression_fn( self.cfg.get_compression_level(), self.cfg.parallel_compression, ); let compressed_stream = compressor(stream); let (mut hashing_reader, _) = streaming_hash::HashingReader::new(compressed_stream); self.upload_file( - &narinfo.url, + &nar_url, &mut hashing_reader, - narinfo.compression.content_type(), - narinfo.compression.content_encoding(), + compression.content_type(), + compression.content_encoding(), ) .await?; @@ -626,15 +686,15 @@ impl S3BinaryCacheClient { if let Ok(file_hash) = Hash::from_slice(harmonia_utils_hash::Algorithm::SHA256, file_hash.as_slice()) { - narinfo.file_hash = Some(file_hash); - narinfo.file_size = Some(file_size as u64); + narinfo.info.download_hash = Some(file_hash); + narinfo.info.download_size = Some(file_size as u64); } // Realisation writing for CA derivations is handled by the caller // (e.g. the queue-runner after resolving and building a CA drv), // not here during path copy. - self.upload_narinfo(store.get_store_dir(), narinfo).await?; + self.upload_narinfo(narinfo).await?; Ok(()) } @@ -642,16 +702,16 @@ impl S3BinaryCacheClient { #[tracing::instrument(skip(self, store, paths), err)] pub async fn copy_paths( &self, - store: &nix_utils::LocalStore, - paths: Vec, + store: &harmonia_store_remote::ConnectionPool, + paths: Vec, repair: bool, ) -> Result<(), CacheError> { use futures::stream::StreamExt as _; let mut stream = tokio_stream::iter(paths) - .map(|p| async move { - tracing::debug!("copying path {p} to s3 binary cache."); - self.copy_path(store, &p, repair).await + .map(|vpi| async move { + tracing::debug!("copying path {} to s3 binary cache.", vpi.path); + self.copy_path(store, &vpi, repair).await }) .buffered(10); @@ -668,13 +728,13 @@ impl S3BinaryCacheClient { #[tracing::instrument(skip(self, realisation), err)] pub async fn write_realisation( &self, - mut realisation: nix_utils::Realisation, + mut realisation: harmonia_store_derivation::realisation::Realisation, ) -> Result<(), CacheError> { let keys = self .signing_keys .iter() .filter_map(|s| s.expose_secret().parse().ok()) - .collect::>(); + .collect::>(); realisation.value.sign_mut(&realisation.key, &keys); let json = serde_json::to_string(&realisation)?; @@ -687,7 +747,7 @@ impl S3BinaryCacheClient { #[tracing::instrument(skip(self), err)] pub async fn download_narinfo( &self, - store_path: &nix_utils::StorePath, + store_path: &StorePath, ) -> Result, CacheError> { if let Some(narinfo) = self.narinfo_cache.get(store_path).await { return Ok(Some(narinfo)); @@ -698,7 +758,7 @@ impl S3BinaryCacheClient { .await? { Some(v) => { - let narinfo: NarInfo = String::from_utf8_lossy(&v).parse()?; + let narinfo = parse_narinfo_txt(&self.cfg.store_dir, std::str::from_utf8(&v)?)?; self.narinfo_cache .insert(store_path.to_owned(), narinfo.clone()) .await; @@ -714,7 +774,7 @@ impl S3BinaryCacheClient { } #[tracing::instrument(skip(self), err)] - pub async fn has_narinfo(&self, store_path: &nix_utils::StorePath) -> Result { + pub async fn has_narinfo(&self, store_path: &StorePath) -> Result { if self.narinfo_cache.contains_key(store_path) { return Ok(true); } @@ -724,24 +784,21 @@ impl S3BinaryCacheClient { #[tracing::instrument(skip(self), err)] pub async fn download_realisation( &self, - id: &nix_utils::DrvOutput, - ) -> Result, CacheError> { - (self.get_object(&format!("realisations/{id}.doi")).await?).map_or_else( - || Ok(None), - |v| Ok(Some(String::from_utf8_lossy(&v).to_string())), - ) + id: &harmonia_store_derivation::realisation::DrvOutput, + ) -> Result, CacheError> { + self.get_object(&format!("realisations/{id}.doi")).await } #[tracing::instrument(skip(self), err)] - pub async fn has_realisation(&self, id: &nix_utils::DrvOutput) -> Result { + pub async fn has_realisation( + &self, + id: &harmonia_store_derivation::realisation::DrvOutput, + ) -> Result { Ok(self.download_realisation(id).await?.is_some()) } #[tracing::instrument(skip(self, paths))] - pub async fn query_missing_paths( - &self, - paths: Vec, - ) -> Vec { + pub async fn query_missing_paths(&self, paths: Vec) -> Vec { use futures::stream::StreamExt as _; tokio_stream::iter(paths) @@ -761,8 +818,8 @@ impl S3BinaryCacheClient { #[tracing::instrument(skip(self, outputs))] pub async fn query_missing_remote_outputs( &self, - outputs: BTreeMap>, - ) -> BTreeMap> { + outputs: BTreeMap>, + ) -> BTreeMap> { use futures::stream::StreamExt as _; tokio_stream::iter(outputs) @@ -781,13 +838,13 @@ impl S3BinaryCacheClient { #[tracing::instrument(skip(self), err)] pub async fn generate_nar_upload_presigned_url( &self, - path: &nix_utils::StorePath, - nix32_nar_hash: &str, + path: &StorePath, + nar_hash: &harmonia_store_path_info::NarHash, debug_info_build_ids: Vec, ) -> Result { - let nar_hash_url = nix32_nar_hash - .strip_prefix("sha256:") - .map_or_else(|| path.hash().to_string(), ToOwned::to_owned); + use harmonia_utils_hash::HashFormat as _; + let h: Hash = (*nar_hash).into(); + let nar_hash_url = format!("{:#}", h.as_base32()); let nar_url = format!("nar/{}.{}", nar_hash_url, self.cfg.compression.ext()); let url = self @@ -877,30 +934,27 @@ impl S3BinaryCacheClient { }) } - #[tracing::instrument(skip(self, store, narinfo), err)] + #[tracing::instrument(skip(self, narinfo), err)] pub async fn upload_narinfo_after_presigned_upload( &self, - store: &nix_utils::LocalStore, narinfo: NarInfo, ) -> Result { if self.cfg.write_nar_listing { - self.head_object(&narinfo.get_ls_path()) + let ls_path = get_ls_path(&narinfo); + self.head_object(&ls_path) .await? .then_some(()) - .ok_or(CacheError::PathNotFound { - path: narinfo.get_ls_path(), - })?; + .ok_or(CacheError::PathNotFound { path: ls_path })?; } - self.head_object(&narinfo.url) + let nar_url = narinfo.info.url.clone().unwrap_or_default(); + self.head_object(&nar_url) .await? .then_some(()) - .ok_or(CacheError::PathNotFound { - path: narinfo.url.clone(), - })?; + .ok_or(CacheError::PathNotFound { path: nar_url })?; - let narinfo = narinfo.clear_sigs_and_sign(store.get_store_dir(), &self.signing_keys); + let narinfo = clear_sigs_and_sign(narinfo, &self.cfg.store_dir, &self.signing_keys); // TODO: we also need to integarte realisation into this! - self.upload_narinfo(store.get_store_dir(), narinfo).await + self.upload_narinfo(narinfo).await } } @@ -954,3 +1008,18 @@ impl debug_info::DebugInfoClient for S3BinaryCacheClient { Ok(()) } } + +/// Generate a NAR listing from a store path via the daemon protocol. +async fn nar_listing( + store: &harmonia_store_remote::ConnectionPool, + path: &StorePath, +) -> Result, CacheError> { + use harmonia_protocol::types::DaemonStore; + + let mut guard = store.acquire().await?; + let nar_reader = guard.client().nar_from_path(path).await?; + let reader = harmonia_utils_io::BytesReader::new(nar_reader); + let listing = harmonia_file_nar::parse_nar_listing(reader).await?; + + Ok(listing) +} diff --git a/subprojects/crates/binary-cache/src/narinfo.rs b/subprojects/crates/binary-cache/src/narinfo.rs index 48a798096..315527ce3 100644 --- a/subprojects/crates/binary-cache/src/narinfo.rs +++ b/subprojects/crates/binary-cache/src/narinfo.rs @@ -1,173 +1,18 @@ -use std::fmt::Write as _; - -use harmonia_store_core::signature::{SecretKey, fingerprint_path}; -use harmonia_store_core::store_path::StoreDir; +use harmonia_store_nar_info::UnkeyedNarInfo; +use harmonia_store_path::{StoreDir, StorePath}; +use harmonia_store_path_info::fingerprint_path; +use harmonia_store_path_info::{NarHash, UnkeyedValidPathInfo}; use harmonia_utils_hash::Hash; -use harmonia_utils_hash::fmt::CommonHash as _; +use harmonia_utils_hash::HashFormat as _; +use harmonia_utils_signature::SecretKey; use secrecy::ExposeSecret as _; use crate::Compression; -#[derive(Debug, Clone)] -pub struct NarInfo { - pub store_path: nix_utils::StorePath, - pub url: String, - pub compression: Compression, - pub file_hash: Option, - pub file_size: Option, - pub nar_hash: Hash, - pub nar_size: u64, - pub references: Vec, - pub deriver: Option, - pub ca: Option, - pub sigs: Vec, -} - -impl NarInfo { - #[must_use] - pub fn new( - path: &nix_utils::StorePath, - path_info: nix_utils::PathInfo, - compression: Compression, - store_dir: &StoreDir, - signing_keys: &[secrecy::SecretString], - ) -> Self { - let nar_hash = parse_hash(&path_info.nar_hash); - let nar_hash_url = nar_hash.as_ref().map_or_else( - || path.hash().to_string(), - |h| format!("{:#}", h.as_base32()), - ); - - let narinfo = Self { - store_path: path.clone(), - url: format!("nar/{}.{}", nar_hash_url, compression.ext()), - compression, - file_hash: None, - file_size: None, - nar_hash: nar_hash.unwrap_or_else(|| { - Hash::from_slice(harmonia_utils_hash::Algorithm::SHA256, &[0; 32]) - .expect("sha256 zero hash") - }), - nar_size: path_info.nar_size, - references: path_info.refs, - deriver: path_info.deriver, - ca: path_info.ca.clone(), - sigs: vec![], - }; - - let mut narinfo = narinfo.clear_sigs_and_sign(store_dir, signing_keys); - if narinfo.sigs.is_empty() && !path_info.sigs.is_empty() { - narinfo.sigs = path_info.sigs; - } - - narinfo - } - - #[must_use] - pub fn simple( - path: &nix_utils::StorePath, - path_info: nix_utils::PathInfo, - compression: Compression, - ) -> Self { - let nar_hash = parse_hash(&path_info.nar_hash); - let nar_hash_url = nar_hash.as_ref().map_or_else( - || path.hash().to_string(), - |h| format!("{:#}", h.as_base32()), - ); - - Self { - store_path: path.clone(), - url: format!("nar/{}.{}", nar_hash_url, compression.ext()), - compression, - file_hash: None, - file_size: None, - nar_hash: nar_hash.unwrap_or_else(|| { - Hash::from_slice(harmonia_utils_hash::Algorithm::SHA256, &[0; 32]) - .expect("sha256 zero hash") - }), - nar_size: path_info.nar_size, - references: path_info.refs, - deriver: path_info.deriver, - ca: path_info.ca, - sigs: vec![], - } - } - - #[must_use] - pub fn clear_sigs_and_sign( - mut self, - store_dir: &StoreDir, - signing_keys: &[secrecy::SecretString], - ) -> Self { - self.sigs.clear(); - if !signing_keys.is_empty() - && let Some(fp) = self.fingerprint(store_dir) - { - for s in signing_keys { - if let Ok(sk) = s.expose_secret().parse::() { - self.sigs.push(sk.sign(&fp).to_string()); - } - } - } - self - } - - #[must_use] - fn fingerprint(&self, store_dir: &StoreDir) -> Option> { - let refs = self.references.iter().cloned().collect(); - let nar_hash_str = format!("{}", self.nar_hash.as_base32()); - fingerprint_path( - store_dir, - &self.store_path, - nar_hash_str.as_bytes(), - self.nar_size, - &refs, - ) - .ok() - } - - #[must_use] - pub fn get_ls_path(&self) -> String { - format!("{}.ls", self.store_path.hash()) - } - - pub fn render(&self, store_dir: &StoreDir) -> Result { - let mut o = String::with_capacity(200); - writeln!(o, "StorePath: {}", store_dir.display(&self.store_path))?; - writeln!(o, "URL: {}", self.url)?; - writeln!(o, "Compression: {}", self.compression.as_str())?; - if let Some(h) = &self.file_hash { - writeln!(o, "FileHash: {}", h.as_base32())?; - } - if let Some(s) = self.file_size { - writeln!(o, "FileSize: {s}")?; - } - writeln!(o, "NarHash: {}", self.nar_hash.as_base32())?; - writeln!(o, "NarSize: {}", self.nar_size)?; - - writeln!( - o, - "References: {}", - self.references - .iter() - .map(nix_utils::StorePath::to_string) - .collect::>() - .join(" ") - )?; +pub use harmonia_store_nar_info::NarInfo; - if let Some(d) = &self.deriver { - writeln!(o, "Deriver: {d}")?; - } - if let Some(ca) = &self.ca { - writeln!(o, "CA: {ca}")?; - } - - for sig in &self.sigs { - writeln!(o, "Sig: {sig}")?; - } - Ok(o) - } -} +/// Re-export the harmonia narinfo formatter and parser. +pub use harmonia_store_nar_info::{format_narinfo_txt, parse_narinfo_txt}; /// Parse a hash string (in any format: hex, nix32, sri) into a typed `Hash`. pub fn parse_hash(raw: &str) -> Option { @@ -176,154 +21,102 @@ pub fn parse_hash(raw: &str) -> Option { .ok() } -#[derive(Debug, thiserror::Error)] -pub enum NarInfoError { - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("invalid value for {field}: {value}")] - InvalidField { field: String, value: String }, - #[error("parse error on line {line}: {reason}")] - Line { line: usize, reason: String }, - #[error("integer parse error for {field}: {err}")] - Int { - field: &'static str, - err: std::num::ParseIntError, - }, +/// Parse a hash string into a `NarHash` (SHA256 only). +#[must_use] +pub fn parse_nar_hash(raw: &str) -> Option { + parse_hash(raw).and_then(|h| NarHash::try_from(h).ok()) } -impl std::str::FromStr for NarInfo { - type Err = NarInfoError; - - #[tracing::instrument(skip(input), err)] - #[allow(clippy::too_many_lines)] - fn from_str(input: &str) -> Result { - let mut out = Self { - store_path: nix_utils::parse_store_path("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bla"), - url: String::new(), - compression: Compression::None, - file_hash: None, - file_size: None, - nar_hash: Hash::from_slice(harmonia_utils_hash::Algorithm::SHA256, &[0; 32]) - .expect("sha256 zero hash"), - nar_size: 0, - references: vec![], - deriver: None, - ca: None, - sigs: vec![], - }; - - // Temporaries to know what was present - let mut have_store_path = false; - let mut have_url = false; - let mut have_compression = false; - let mut have_nar_hash = false; - let mut have_nar_size = false; +/// Build a `NarInfo` from a `PathInfo` (`UnkeyedValidPathInfo`), optionally signing it. +#[must_use] +pub fn narinfo_from_path_info( + path: &StorePath, + path_info: UnkeyedValidPathInfo, + compression: Compression, + store_dir: &StoreDir, + signing_keys: &[secrecy::SecretString], +) -> NarInfo { + let nar_hash_url = { + let h: Hash = path_info.nar_hash.into(); + format!("{:#}", h.as_base32()) + }; + + let original_signatures = path_info.signatures.clone(); + let url = format!("nar/{}.{}", nar_hash_url, compression.ext()); + + let mut narinfo = NarInfo { + path: path.clone(), + info: UnkeyedNarInfo { + info: path_info, + url: Some(url), + compression: Some(compression.as_str().to_owned()), + download_hash: None, + download_size: None, + }, + }; + + // Sign with the provided signing keys (clears existing sigs first) + narinfo = clear_sigs_and_sign(narinfo, store_dir, signing_keys); + + // If signing produced no sigs but path_info had sigs, restore them + if narinfo.info.info.signatures.is_empty() && !original_signatures.is_empty() { + narinfo.info.info.signatures = original_signatures; + } - for (idx, raw_line) in input.lines().enumerate() { - let line_no = idx + 1; - let line = raw_line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } + narinfo +} - let Some((k, v)) = line.split_once(':') else { - return Err(NarInfoError::Line { - line: line_no, - reason: "expected `Key: value`".into(), - }); - }; - let key = k.trim(); - let val = v - .strip_prefix(' ') - .map_or(v, |stripped| stripped) - .trim_end(); +/// Build a simple `NarInfo` without signing. +#[must_use] +pub fn narinfo_simple( + path: &StorePath, + path_info: UnkeyedValidPathInfo, + compression: Compression, +) -> NarInfo { + let nar_hash_url = { + let h: Hash = path_info.nar_hash.into(); + format!("{:#}", h.as_base32()) + }; + + NarInfo { + path: path.clone(), + info: UnkeyedNarInfo { + info: path_info, + url: Some(format!("nar/{}.{}", nar_hash_url, compression.ext())), + compression: Some(compression.as_str().to_owned()), + download_hash: None, + download_size: None, + }, + } +} - match key { - "StorePath" => { - out.store_path = nix_utils::parse_store_path(val); - have_store_path = true; - } - "URL" => { - out.url = val.to_string(); - have_url = true; - } - "Compression" => { - out.compression = val.parse().map_err(|e| NarInfoError::InvalidField { - field: "Compression".into(), - value: e, - })?; - have_compression = true; - } - "FileHash" => { - out.file_hash = parse_hash(val); - } - "FileSize" => { - out.file_size = Some(val.parse::().map_err(|e| NarInfoError::Int { - field: "FileSize", - err: e, - })?); - } - "NarHash" => { - if let Some(h) = parse_hash(val) { - out.nar_hash = h; - have_nar_hash = true; - } - } - "NarSize" => { - out.nar_size = val.parse::().map_err(|e| NarInfoError::Int { - field: "NarSize", - err: e, - })?; - have_nar_size = true; - } - "References" => { - let refs = val - .split_whitespace() - .filter(|s| !s.is_empty()) - .map(nix_utils::parse_store_path) - .collect::>(); - out.references = refs; - } - "Deriver" => { - out.deriver = if val.is_empty() { - None - } else { - Some(nix_utils::parse_store_path(val)) - }; - } - "CA" => { - out.ca = if val.is_empty() { - None - } else { - Some(val.to_string()) - }; - } - "Sig" => { - if !val.is_empty() { - out.sigs.push(val.to_string()); - } - } - _ => {} +/// Clear signatures and re-sign with the provided signing keys. +#[must_use] +pub fn clear_sigs_and_sign( + mut narinfo: NarInfo, + store_dir: &StoreDir, + signing_keys: &[secrecy::SecretString], +) -> NarInfo { + narinfo.info.info.signatures.clear(); + if !signing_keys.is_empty() { + let fp = fingerprint_path( + store_dir, + &narinfo.path, + &narinfo.info.info.nar_hash, + narinfo.info.info.nar_size, + &narinfo.info.info.references, + ); + for s in signing_keys { + if let Ok(sk) = s.expose_secret().parse::() { + narinfo.info.info.signatures.insert(sk.sign(&fp)); } } - - // Validate requireds - if !have_store_path { - return Err(NarInfoError::MissingField("StorePath")); - } - if !have_url { - return Err(NarInfoError::MissingField("URL")); - } - if !have_compression { - return Err(NarInfoError::MissingField("Compression")); - } - if !have_nar_hash { - return Err(NarInfoError::MissingField("NarHash")); - } - if !have_nar_size { - return Err(NarInfoError::MissingField("NarSize")); - } - - Ok(out) } + narinfo +} + +/// Return the `.ls` listing key for this narinfo. +#[must_use] +pub fn get_ls_path(narinfo: &NarInfo) -> String { + format!("{}.ls", narinfo.path.hash()) } diff --git a/subprojects/crates/binary-cache/src/presigned.rs b/subprojects/crates/binary-cache/src/presigned.rs index 3b5c13c68..3e5d828e8 100644 --- a/subprojects/crates/binary-cache/src/presigned.rs +++ b/subprojects/crates/binary-cache/src/presigned.rs @@ -1,13 +1,10 @@ use std::sync::atomic::{AtomicU64, Ordering}; use backon::Retryable; - use bytes::Bytes; -use nix_utils::{BaseStore as _, LocalStore}; - -use tokio_util::io::StreamReader; +use harmonia_store_path::StorePath; -use crate::{CacheError, Compression, streaming_hash::HashingReader}; +use crate::{CacheError, Compression, read_nar_stream, streaming_hash::HashingReader}; const RETRY_MIN_DELAY_SECS: u64 = 1; const RETRY_MAX_DELAY_SECS: u64 = 30; @@ -108,17 +105,15 @@ impl PresignedUploadClient { #[tracing::instrument(skip(self, store, narinfo, req), err)] pub async fn process_presigned_request( &self, - store: &LocalStore, + store: &harmonia_store_remote::ConnectionPool, mut narinfo: crate::NarInfo, req: PresignedUploadResponse, ) -> Result { - narinfo.url = req.nar_url; - narinfo.compression = req.nar_upload.compression; + narinfo.info.url = Some(req.nar_url.clone()); + narinfo.info.compression = Some(req.nar_upload.compression.as_str().to_owned()); if let Some(ls_upload) = req.ls_upload { - let _ = self - .upload_ls(store, &narinfo.store_path, &ls_upload) - .await?; + let _ = self.upload_ls(store, &narinfo.path, &ls_upload).await?; } if !req.debug_info_upload.is_empty() { @@ -127,19 +122,19 @@ impl PresignedUploadClient { debug_info_urls: std::sync::Arc::new(req.debug_info_upload), }; crate::debug_info::process_debug_info( - &narinfo.url, - store, - &narinfo.store_path, + &req.nar_url, + store.store_dir().as_ref(), + &narinfo.path, debug_info_client.clone(), ) .await?; } let upload_res = self - .upload_nar(store, &narinfo.store_path, &req.nar_upload) + .upload_nar(store, &narinfo.path, &req.nar_upload) .await?; - narinfo.file_hash = Some(upload_res.file_hash); - narinfo.file_size = Some(upload_res.file_size); + narinfo.info.download_hash = Some(upload_res.file_hash); + narinfo.info.download_size = Some(upload_res.file_size); Ok(narinfo) } @@ -147,68 +142,37 @@ impl PresignedUploadClient { #[tracing::instrument(skip(self, store, store_path), err)] async fn upload_nar( &self, - store: &LocalStore, - store_path: &nix_utils::StorePath, + store: &harmonia_store_remote::ConnectionPool, + store_path: &StorePath, upload: &PresignedUpload, ) -> Result { let start = std::time::Instant::now(); - let (tx, rx) = tokio::sync::mpsc::unbounded_channel::>(); - let (result_tx, result_rx) = tokio::sync::oneshot::channel::>(); - - let closure = { - let tx = tx.clone(); - move |data: &[u8]| { - let data = Bytes::copy_from_slice(data); - tx.send(Ok(data)).is_ok() - } - }; - - tokio::task::spawn({ - let path = store_path.clone(); - let store = store.clone(); - async move { - let result = store - .nar_from_path(&path, closure) - .map_err(|e| format!("NAR reading failed: {e}")); - let _ = result_tx.send(result); - } - }); - - drop(tx); - let stream = StreamReader::new(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)); + let stream = read_nar_stream(store, store_path); let compressor = upload .compression .get_compression_fn(upload.compression_level, false); let compressed_stream = compressor(stream); let (hashing_reader, _) = HashingReader::new(compressed_stream); - let upload_result = self.upload_any(upload, hashing_reader, start, None).await; - - match result_rx.await { - Ok(Ok(())) => upload_result, - Ok(Err(e)) => Err(CacheError::UploadError { - path: upload.path.clone(), - reason: e, - }), - Err(_) => Err(CacheError::UploadError { - path: upload.path.clone(), - reason: "NAR reading task was cancelled or panicked".to_string(), - }), - } + self.upload_any(upload, hashing_reader, start, None).await } - #[tracing::instrument(skip(self, store, store_path), err)] + #[tracing::instrument(skip(self, store), err)] async fn upload_ls( &self, - store: &LocalStore, - store_path: &nix_utils::StorePath, + store: &harmonia_store_remote::ConnectionPool, + path: &StorePath, upload: &PresignedUpload, ) -> Result { let start = std::time::Instant::now(); - let ls = store.list_nar_deep(store_path).await?; - let stream = Box::new(std::io::Cursor::new(Bytes::from(ls))); + let listing = super::nar_listing(store, path).await?; + let ls_json = serde_json::json!({ + "version": 1, + "root": listing, + }); + let stream = Box::new(std::io::Cursor::new(Bytes::from(ls_json.to_string()))); let compressor = upload .compression .get_compression_fn(upload.compression_level, false); @@ -246,64 +210,74 @@ impl PresignedUploadClient { start: std::time::Instant, content_type: Option<&str>, ) -> Result { - use tokio::io::AsyncReadExt as _; + // Clippy has a false positive and suggests using blocks, + // but that would not allow processing errors from the ? operator to add context + #[allow(clippy::redundant_closure_call)] + async move || -> Result { + use tokio::io::AsyncReadExt as _; + + let mut request = self.client.put(&upload.url); + if let Some(content_type) = content_type { + request = request.header("Content-Type", content_type); + } else { + request = request.header("Content-Type", upload.compression.content_type()); + } + if !upload.compression.content_encoding().is_empty() { + request = request.header("Content-Encoding", upload.compression.content_encoding()); + } - let mut request = self.client.put(&upload.url); - if let Some(content_type) = content_type { - request = request.header("Content-Type", content_type); - } else { - request = request.header("Content-Type", upload.compression.content_type()); - } - if !upload.compression.content_encoding().is_empty() { - request = request.header("Content-Encoding", upload.compression.content_encoding()); - } + // TODO: We need multipart signed urls to fix this! + // object_store currently doesnt have support for this. + let mut buffer = Vec::new(); + reader.read_to_end(&mut buffer).await?; + + let _response = (|| async { + Ok::<_, CacheError>( + request + .try_clone() + .ok_or_else(|| CacheError::RequestCloneError)? + .body(buffer.clone()) + .send() + .await? + .error_for_status()?, + ) + }) + .retry( + &backon::ExponentialBuilder::default() + .with_min_delay(std::time::Duration::from_secs(RETRY_MIN_DELAY_SECS)) + .with_max_delay(std::time::Duration::from_secs(RETRY_MAX_DELAY_SECS)) + .with_max_times(RETRY_MAX_ATTEMPTS), + ) + .await?; + + let elapsed = u64::try_from(start.elapsed().as_millis()).unwrap_or_default(); - // TODO: We need multipart signed urls to fix this! - // object_store currently doesnt have support for this. - let mut buffer = Vec::new(); - reader.read_to_end(&mut buffer).await?; - - let _response = (|| async { - Ok::<_, CacheError>( - request - .try_clone() - .ok_or_else(|| CacheError::RequestCloneError)? - .body(buffer.clone()) - .send() - .await? - .error_for_status()?, + let (file_hash, file_size) = reader.finalize()?; + + let file_hash = harmonia_utils_hash::Hash::from_slice( + harmonia_utils_hash::Algorithm::SHA256, + file_hash.as_slice(), ) - }) - .retry( - &backon::ExponentialBuilder::default() - .with_min_delay(std::time::Duration::from_secs(RETRY_MIN_DELAY_SECS)) - .with_max_delay(std::time::Duration::from_secs(RETRY_MAX_DELAY_SECS)) - .with_max_times(RETRY_MAX_ATTEMPTS), - ) - .await?; - - let elapsed = u64::try_from(start.elapsed().as_millis()).unwrap_or_default(); - - let (file_hash, file_size) = reader.finalize()?; - - let file_hash = harmonia_utils_hash::Hash::from_slice( - harmonia_utils_hash::Algorithm::SHA256, - file_hash.as_slice(), - ) - .map_err(|e| CacheError::Signing(format!("invalid file hash: {e}")))?; - - // Update metrics - self.metrics - .put_bytes - .fetch_add(file_size as u64, Ordering::Relaxed); - self.metrics - .put_time_ms - .fetch_add(elapsed, Ordering::Relaxed); - self.metrics.put.fetch_add(1, Ordering::Relaxed); - - Ok(PresignedUploadResult { - file_hash, - file_size: file_size as u64, + .map_err(|e| CacheError::Signing(format!("invalid file hash: {e}")))?; + + // Update metrics + self.metrics + .put_bytes + .fetch_add(file_size as u64, Ordering::Relaxed); + self.metrics + .put_time_ms + .fetch_add(elapsed, Ordering::Relaxed); + self.metrics.put.fetch_add(1, Ordering::Relaxed); + + Ok(PresignedUploadResult { + file_hash, + file_size: file_size as u64, + }) + }() + .await + .map_err(|source| CacheError::Upload { + path: upload.url.clone(), + source: source.into(), }) } } @@ -351,9 +325,9 @@ impl crate::debug_info::DebugInfoClient for PresignedDebugInfoUpload { .debug_info_urls .iter() .find(|presigned| presigned.path == key) - .ok_or(CacheError::UploadError { + .ok_or(CacheError::PresignedUrlError { path: key.clone(), - reason: format!("Presigned URL not found for build ID: {build_id}"), + reason: "Presigned URL not found".to_string(), })?; let json_content = crate::debug_info::DebugInfoLink { diff --git a/subprojects/crates/daemon-client-utils/Cargo.toml b/subprojects/crates/daemon-client-utils/Cargo.toml new file mode 100644 index 000000000..d23a09aa7 --- /dev/null +++ b/subprojects/crates/daemon-client-utils/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "daemon-client-utils" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +petgraph.workspace = true + +harmonia-protocol.workspace = true +harmonia-store-derivation.workspace = true +harmonia-store-path.workspace = true +harmonia-store-path-info.workspace = true +harmonia-store-remote.workspace = true +url.workspace = true diff --git a/subprojects/crates/daemon-client-utils/src/lib.rs b/subprojects/crates/daemon-client-utils/src/lib.rs new file mode 100644 index 000000000..9e15ce128 --- /dev/null +++ b/subprojects/crates/daemon-client-utils/src/lib.rs @@ -0,0 +1,403 @@ +//! Utilities for working with the Nix daemon beyond what the +//! harmonia libraries provide. + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use harmonia_protocol::types::{DaemonError, DaemonStore}; +use harmonia_store_path::{StoreDir, StorePath}; +use harmonia_store_path_info::ValidPathInfo; +use harmonia_store_remote::ConnectionPool; + +/// Parsed nix daemon store connection settings. +/// +/// Constructed by [`parse_nix_remote`] from `NIX_REMOTE` and fallback +/// env vars. Precedence rules match nix's `LocalFSStoreConfig`: +/// URI query params (`?store=`, `?root=`, `?state=`, `?real=`) +/// override env vars; `?root=` derives default state and real store +/// dirs. +/// +/// Use [`to_uri`](Self::to_uri) to reconstruct a `unix://` URI +/// suitable for `nix copy --from` etc. +pub struct NixDaemonStoreConfig { + /// Path to the daemon socket. + pub socket: String, + /// Logical store directory (e.g. `/nix/store`). + pub store_dir: StoreDir, + /// Chroot root directory, if any (e.g. `/foo`). A physical path. + pub root: Option, + /// Explicit physical store directory override (`real` query param). + real: Option, + /// Nix state directory (e.g. `/nix/var/nix`). + pub state_dir: PathBuf, +} + +impl NixDaemonStoreConfig { + /// Physical store directory on disk, if it differs from the + /// logical store dir. + /// + /// Matches nix's `LocalFSStoreConfig::realStoreDir`: + /// - `?real=` if explicitly set + /// - `root / "nix/store"` if `?root=` is set (hardcoded, not derived from `store`) + /// - `None` otherwise (callers should use `store_dir`) + pub fn real_store_dir(&self) -> Option { + if let Some(ref real) = self.real { + Some(real.clone()) + } else { + self.root.as_ref().map(|root| root.join("nix/store")) + } + } + + /// Reconstruct a `unix://` URI suitable for `nix copy --from` etc. + pub fn to_uri(&self) -> String { + let mut uri = format!("unix://{}", self.socket); + let mut params = Vec::new(); + let store_str = self.store_dir.to_string(); + if store_str != "/nix/store" { + params.push(format!("store={store_str}")); + } + if let Some(ref root) = self.root { + params.push(format!("root={}", root.display())); + } + if let Some(ref real) = self.real { + params.push(format!("real={}", real.display())); + } + if !params.is_empty() { + uri.push('?'); + uri.push_str(¶ms.join("&")); + } + uri + } +} + +/// Default nix state directory relative path (under root). +const DEFAULT_STATE_DIR_RELATIVE: &str = "nix/var/nix"; + +/// Parse daemon store settings from `NIX_REMOTE` and related env vars. +pub fn parse_nix_remote() -> Result { + parse_nix_remote_from( + std::env::var("NIX_REMOTE").ok().as_deref(), + std::env::var("NIX_STORE_DIR").ok().as_deref(), + std::env::var("NIX_STATE_DIR").ok().as_deref(), + std::env::var("NIX_DAEMON_SOCKET_PATH").ok().as_deref(), + ) +} + +/// Factored out pure internal function for unit testing purposes. +fn parse_nix_remote_from( + nix_remote: Option<&str>, + nix_store_dir: Option<&str>, + nix_state_dir: Option<&str>, + nix_daemon_socket_path: Option<&str>, +) -> Result { + let explicit_state_dir = nix_state_dir.map(String::from); + let explicit_socket = nix_daemon_socket_path.map(String::from); + let mut socket_from_uri = None; + let mut state_from_uri = None; + let mut store = nix_store_dir + .map(String::from) + .unwrap_or_else(|| "/nix/store".to_owned()); + let mut root = None; + let mut real = None; + + // NIX_REMOTE URL query params override env vars. + if let Some(remote) = nix_remote + && remote.starts_with("unix://") + { + let parsed = url::Url::parse(remote).map_err(|e| e.to_string())?; + let path = parsed.path(); + if !path.is_empty() { + socket_from_uri = Some(path.to_owned()); + } + + for (k, v) in parsed.query_pairs() { + match k.as_ref() { + "store" => store = v.into_owned(), + "root" => root = Some(PathBuf::from(v.as_ref())), + "real" => real = Some(PathBuf::from(v.as_ref())), + "state" => state_from_uri = Some(PathBuf::from(v.as_ref())), + _ => {} + } + } + } + + // Derive state_dir matching nix's LocalFSStoreConfig::stateDir: + // ?state= > (root / "nix/var/nix" if root set, else NIX_STATE_DIR) + let state_dir = if let Some(s) = state_from_uri { + s + } else if let Some(ref r) = root { + r.join(DEFAULT_STATE_DIR_RELATIVE) + } else if let Some(s) = explicit_state_dir { + PathBuf::from(s) + } else { + PathBuf::from("/").join(DEFAULT_STATE_DIR_RELATIVE) + }; + + // Derive socket. URI path overrides everything; NIX_DAEMON_SOCKET_PATH + // is a fallback for when the URI has no path (e.g. `unix://?store=...`). + // Final default: stateDir / "daemon-socket/socket". + let socket = match socket_from_uri.or(explicit_socket) { + Some(s) => s, + None => state_dir + .join("daemon-socket/socket") + .into_os_string() + .into_string() + .map_err(|p| format!("derived socket path is not valid UTF-8: {p:?}"))?, + }; + + Ok(NixDaemonStoreConfig { + socket, + store_dir: StoreDir::new(store).map_err(|e| e.to_string())?, + root, + real, + state_dir, + }) +} + +/// Walk store path references transitively, collecting all path infos +/// in the closure. No particular ordering is guaranteed. +async fn walk_closure( + pool: &ConnectionPool, + roots: &[StorePath], +) -> Result, DaemonError> { + let mut infos = HashMap::new(); + let mut queue: Vec = roots.to_vec(); + let mut visited = HashSet::new(); + while let Some(p) = queue.pop() { + if !visited.insert(p.clone()) { + continue; + } + let mut guard = pool.acquire().await?; + let info = guard + .client() + .query_path_info(&p) + .await? + .ok_or_else(|| DaemonError::custom(format!("path '{p}' is not valid")))?; + for r in &info.references { + if !visited.contains(r) { + queue.push(r.clone()); + } + } + infos.insert(p, info); + } + Ok(infos) +} + +/// Walk store path references transitively to compute the closure. +/// +/// Returns `ValidPathInfo`s topologically sorted (dependencies before +/// dependents) via `petgraph`. This ordering is required by +/// `add_to_store_nar`, which validates that all references already +/// exist before accepting a path. +pub async fn query_closure( + pool: &ConnectionPool, + roots: &[StorePath], +) -> Result, DaemonError> { + use petgraph::graphmap::DiGraphMap; + + let infos = walk_closure(pool, roots).await?; + + // Topological sort so dependencies come before dependents. + let mut graph = DiGraphMap::<&StorePath, ()>::new(); + for p in infos.keys() { + graph.add_node(p); + } + for (p, info) in &infos { + for r in &info.references { + if r != p && infos.contains_key(r) { + graph.add_edge(p, r, ()); + } + } + } + + // petgraph toposort returns dependents before dependencies, so + // reverse to get dependencies first. + let sorted = + petgraph::algo::toposort(&graph, None).expect("store reference graph should be acyclic"); + Ok(sorted + .into_iter() + .rev() + .filter_map(|p| { + Some(ValidPathInfo { + path: p.clone(), + info: infos.get(p)?.clone(), + }) + }) + .collect()) +} + +/// Compute the total NAR size of a path's closure. +pub async fn compute_closure_size(pool: &ConnectionPool, path: &StorePath) -> u64 { + walk_closure(pool, std::slice::from_ref(path)) + .await + .map(|infos| infos.values().map(|info| info.nar_size).sum()) + .unwrap_or(0) +} + +/// Check whether a store path is valid. +pub async fn is_valid_path(pool: &ConnectionPool, path: &StorePath) -> Result { + let mut guard = pool.acquire().await?; + guard.client().is_valid_path(path).await +} + +/// Query path info, returning `None` if the path is not valid. +pub async fn query_path_info( + pool: &ConnectionPool, + path: &StorePath, +) -> Result, DaemonError> { + let mut guard = pool.acquire().await?; + guard.client().query_path_info(path).await +} + +/// Ensure a path is present in the store (via substitution). +pub async fn ensure_path(pool: &ConnectionPool, path: &StorePath) -> Result<(), DaemonError> { + let mut guard = pool.acquire().await?; + guard.client().ensure_path(path).await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse( + remote: Option<&str>, + store_dir: Option<&str>, + state_dir: Option<&str>, + socket_path: Option<&str>, + ) -> NixDaemonStoreConfig { + parse_nix_remote_from(remote, store_dir, state_dir, socket_path).unwrap() + } + + #[test] + fn defaults() { + let c = parse(None, None, None, None); + assert_eq!(c.socket, "/nix/var/nix/daemon-socket/socket"); + assert_eq!(c.store_dir.to_string(), "/nix/store"); + assert_eq!(c.state_dir, PathBuf::from("/nix/var/nix")); + assert_eq!(c.real_store_dir(), None); + assert_eq!(c.root, None); + } + + #[test] + fn env_overrides() { + let c = parse( + None, + Some("/custom/store"), + Some("/custom/state"), + Some("/custom/socket"), + ); + assert_eq!(c.socket, "/custom/socket"); + assert_eq!(c.store_dir.to_string(), "/custom/store"); + assert_eq!(c.state_dir, PathBuf::from("/custom/state")); + assert_eq!(c.real_store_dir(), None); + } + + #[test] + fn unix_uri_socket_path() { + let c = parse(Some("unix:///run/nix/socket"), None, None, None); + assert_eq!(c.socket, "/run/nix/socket"); + // URI path overrides NIX_DAEMON_SOCKET_PATH + let c = parse( + Some("unix:///run/nix/socket"), + None, + None, + Some("/other/socket"), + ); + assert_eq!(c.socket, "/run/nix/socket"); + } + + #[test] + fn unix_uri_empty_path_falls_back_to_env() { + let c = parse(Some("unix://?store=/foo"), None, None, Some("/env/socket")); + assert_eq!(c.socket, "/env/socket"); + } + + #[test] + fn root_derives_state_and_real_store() { + let c = parse(Some("unix:///sock?root=/chroot"), None, None, None); + assert_eq!(c.state_dir, PathBuf::from("/chroot/nix/var/nix")); + assert_eq!(c.real_store_dir(), Some(PathBuf::from("/chroot/nix/store"))); + assert_eq!(c.store_dir.to_string(), "/nix/store"); + } + + #[test] + fn root_overrides_nix_state_dir() { + // root takes precedence over NIX_STATE_DIR for state_dir + let c = parse( + Some("unix:///sock?root=/chroot"), + None, + Some("/env/state"), + None, + ); + assert_eq!(c.state_dir, PathBuf::from("/chroot/nix/var/nix")); + } + + #[test] + fn explicit_state_param_overrides_root() { + let c = parse( + Some("unix:///sock?root=/chroot&state=/explicit/state"), + None, + None, + None, + ); + assert_eq!(c.state_dir, PathBuf::from("/explicit/state")); + } + + #[test] + fn explicit_real_overrides_root_derived() { + let c = parse( + Some("unix:///sock?root=/chroot&real=/explicit/real"), + None, + None, + None, + ); + assert_eq!(c.real_store_dir(), Some(PathBuf::from("/explicit/real"))); + } + + #[test] + fn root_with_custom_store_still_uses_nix_store_for_real() { + // Even with a custom logical store, realStoreDir is root/"nix/store" + // (matching nix's hardcoded default) + let c = parse( + Some("unix:///sock?root=/chroot&store=/custom/store"), + None, + None, + None, + ); + assert_eq!(c.store_dir.to_string(), "/custom/store"); + assert_eq!(c.real_store_dir(), Some(PathBuf::from("/chroot/nix/store"))); + } + + #[test] + fn socket_derived_from_state_dir() { + // No URI, no NIX_DAEMON_SOCKET_PATH — socket comes from state_dir + let c = parse(None, None, Some("/custom/state"), None); + assert_eq!(c.socket, "/custom/state/daemon-socket/socket"); + } + + #[test] + fn socket_derived_from_root_state_dir() { + // root derives state_dir, which derives socket + let c = parse(Some("unix://?root=/chroot"), None, None, None); + assert_eq!(c.socket, "/chroot/nix/var/nix/daemon-socket/socket"); + } + + #[test] + fn to_uri_roundtrip_default() { + let c = parse(None, None, None, None); + assert_eq!(c.to_uri(), "unix:///nix/var/nix/daemon-socket/socket"); + } + + #[test] + fn to_uri_with_root_and_store() { + let c = parse( + Some("unix:///sock?root=/chroot&store=/custom/store"), + None, + None, + None, + ); + let uri = c.to_uri(); + assert!(uri.contains("store=/custom/store")); + assert!(uri.contains("root=/chroot")); + } +} diff --git a/subprojects/crates/db/Cargo.toml b/subprojects/crates/db/Cargo.toml index 3b7f2507f..f9c33c112 100644 --- a/subprojects/crates/db/Cargo.toml +++ b/subprojects/crates/db/Cargo.toml @@ -6,11 +6,15 @@ license = "GPL-3.0" rust-version.workspace = true [dependencies] -anyhow.workspace = true -futures.workspace = true -harmonia-store-core.workspace = true -hashbrown.workspace = true -tracing.workspace = true +futures.workspace = true +harmonia-store-derivation.workspace = true +harmonia-store-path.workspace = true +harmonia-utils-hash.workspace = true +hashbrown.workspace = true +nix-support.workspace = true +store-path-utils.workspace = true +thiserror.workspace = true +tracing.workspace = true jiff.workspace = true serde_json.workspace = true @@ -18,5 +22,5 @@ serde_json.workspace = true sqlx = { workspace = true, features = [ "runtime-tokio", "tls-rustls-ring-webpki", "postgres" ] } [dev-dependencies] -test-utils = { path = "../test-utils" } -tokio = { workspace = true, features = [ "macros" ] } +test-utils.workspace = true +tokio = { workspace = true, features = [ "macros" ] } diff --git a/subprojects/crates/db/src/connection.rs b/subprojects/crates/db/src/connection.rs index 6b3f1ea81..7efba5a81 100644 --- a/subprojects/crates/db/src/connection.rs +++ b/subprojects/crates/db/src/connection.rs @@ -1,10 +1,10 @@ use std::collections::BTreeMap; +use std::fmt::Write as _; -use anyhow::Context; use sqlx::Acquire; -use harmonia_store_core::derived_path::OutputName; -use harmonia_store_core::store_path::{StoreDir, StorePath}; +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_path::{StoreDir, StorePath}; use super::models::{ Build, BuildSmall, BuildStatus, BuildSteps, InsertBuildMetric, InsertBuildProduct, @@ -29,14 +29,14 @@ impl Connection { } #[tracing::instrument(skip(self), err)] - pub async fn begin_transaction(&mut self) -> sqlx::Result> { + pub async fn begin_transaction(&mut self) -> crate::Result> { let tx = self.conn.begin().await?; Ok(Transaction { tx }) } #[tracing::instrument(skip(self), err)] - pub async fn get_not_finished_builds_fast(&mut self) -> sqlx::Result> { - sqlx::query_as!( + pub async fn get_not_finished_builds_fast(&mut self) -> crate::Result> { + Ok(sqlx::query_as!( BuildSmall, r#" SELECT @@ -46,14 +46,14 @@ impl Connection { WHERE finished = 0;"# ) .fetch_all(&mut *self.conn) - .await + .await?) } #[tracing::instrument(skip(self), err)] pub async fn get_not_finished_builds( &mut self, store_dir: &StoreDir, - ) -> anyhow::Result> { + ) -> crate::Result> { let rows = sqlx::query_as!( Build::, r#" @@ -81,8 +81,8 @@ impl Connection { } #[tracing::instrument(skip(self), err)] - pub async fn get_jobsets(&mut self) -> sqlx::Result> { - sqlx::query_as!( + pub async fn get_jobsets(&mut self) -> crate::Result> { + Ok(sqlx::query_as!( Jobset, r#" SELECT @@ -92,21 +92,22 @@ impl Connection { FROM jobsets"# ) .fetch_all(&mut *self.conn) - .await + .await?) } #[tracing::instrument(skip(self), err)] pub async fn get_jobset_scheduling_shares( &mut self, jobset_id: i32, - ) -> sqlx::Result> { + ) -> crate::Result> { Ok(sqlx::query!( "SELECT schedulingshares FROM jobsets WHERE id = $1", jobset_id, ) .fetch_optional(&mut *self.conn) .await? - .map(|v| v.schedulingshares)) + .map(|v| u32::try_from(v.schedulingshares)) + .transpose()?) } #[tracing::instrument(skip(self), err)] @@ -114,9 +115,9 @@ impl Connection { &mut self, jobset_id: i32, scheduling_window: i64, - ) -> sqlx::Result> { + ) -> crate::Result> { #[allow(clippy::cast_precision_loss)] - sqlx::query_as!( + Ok(sqlx::query_as!( BuildSteps, r#" SELECT s.startTime, s.stopTime FROM buildsteps s join builds b on build = id @@ -129,11 +130,14 @@ impl Connection { jobset_id, ) .fetch_all(&mut *self.conn) - .await + .await?) } + // TODO Currently unused. In the old C++ queue-runner, this was called + // in queue-monitor.cc to mark GC'ed builds as aborted. The Rust + // queue runner apparently doesn't handle that case yet. #[tracing::instrument(skip(self), err)] - pub async fn abort_build(&mut self, build_id: i32) -> sqlx::Result<()> { + pub async fn abort_build(&mut self, build_id: i32) -> crate::Result<()> { #[allow(clippy::cast_possible_truncation)] sqlx::query!( "UPDATE builds SET finished = 1, buildStatus = $2, startTime = $3, stopTime = $3 where id = $1 and finished = 0", @@ -152,7 +156,7 @@ impl Connection { &mut self, store_dir: &StoreDir, paths: &[StorePath], - ) -> sqlx::Result { + ) -> crate::Result { let paths: Vec = paths .iter() .map(|p| store_dir.display(p).to_string()) @@ -166,7 +170,7 @@ impl Connection { } #[tracing::instrument(skip(self), err)] - pub async fn clear_busy(&mut self, stop_time: i32) -> sqlx::Result<()> { + pub async fn clear_busy(&mut self, stop_time: i32) -> crate::Result<()> { sqlx::query!( "UPDATE buildsteps SET busy = 0, status = $1, stopTime = $2 WHERE busy != 0;", BuildStatus::Aborted as i32, @@ -178,7 +182,7 @@ impl Connection { } #[tracing::instrument(skip(self, step), err)] - pub async fn update_build_step(&mut self, step: UpdateBuildStep) -> sqlx::Result<()> { + pub async fn update_build_step(&mut self, step: UpdateBuildStep) -> crate::Result<()> { sqlx::query!( "UPDATE buildsteps SET busy = $1 WHERE build = $2 AND stepnr = $3 AND busy != 0 AND status IS NULL", step.status as i32, @@ -196,7 +200,7 @@ impl Connection { jobset_id: i32, drv_path: &StorePath, system: &str, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { let drv_path = store_dir.display(drv_path).to_string(); sqlx::query!( r#"INSERT INTO builds ( @@ -242,9 +246,9 @@ impl Connection { &mut self, store_dir: &StoreDir, out_path: &StorePath, - ) -> sqlx::Result> { + ) -> crate::Result> { let out_path = store_dir.display(out_path).to_string(); - sqlx::query_as!( + Ok(sqlx::query_as!( super::models::BuildOutput, r#" SELECT @@ -255,18 +259,20 @@ impl Connection { out_path.as_str(), ) .fetch_optional(&mut *self.conn) - .await + .await?) } pub async fn get_build_products_for_build_id( &mut self, build_id: i32, store_dir: &StoreDir, - ) -> anyhow::Result> { + ) -> crate::Result> { let rows = sqlx::query_as!( - crate::models::OwnedBuildProduct::, + crate::models::BuildProductRow, r#" SELECT + build, + productnr, type, subtype, fileSize, @@ -281,15 +287,15 @@ impl Connection { .fetch_all(&mut *self.conn) .await?; rows.into_iter() - .map(|r| Ok(r.parse_paths(store_dir)?)) + .map(|r| Ok(r.into_build_product(store_dir)?)) .collect() } pub async fn get_build_metrics_for_build_id( &mut self, build_id: i32, - ) -> sqlx::Result> { - sqlx::query_as!( + ) -> crate::Result> { + let rows = sqlx::query_as!( crate::models::OwnedBuildMetric, r#" SELECT @@ -299,7 +305,8 @@ impl Connection { build_id ) .fetch_all(&mut *self.conn) - .await + .await?; + Ok(rows.into_iter().map(Into::into).collect()) } /// Resolve output paths for derivation chains via `buildstepoutputs`. @@ -317,7 +324,7 @@ impl Connection { &mut self, store_dir: &StoreDir, chains: &[(&StorePath, &[&OutputName])], - ) -> sqlx::Result>> { + ) -> crate::Result>> { if chains.is_empty() { return Ok(Vec::new()); } @@ -381,16 +388,8 @@ impl Connection { let mut results = vec![None; chains.len()]; for (idx, path) in rows { - let i = usize::try_from(idx - 1) - .context("SQL ordinality is always positive") - .map_err(|e| sqlx::Error::Decode(e.into_boxed_dyn_error()))?; - results[i] = path - .map(|p| { - store_dir - .parse(&p) - .map_err(|e| sqlx::Error::Decode(Box::new(e))) - }) - .transpose()?; + let i = usize::try_from(idx - 1)?; + results[i] = path.map(|p| store_dir.parse(&p)).transpose()?; } Ok(results) } @@ -402,7 +401,7 @@ impl Connection { store_dir: &StoreDir, drv_path: &StorePath, output_name: &OutputName, - ) -> sqlx::Result> { + ) -> crate::Result> { let drv_display = store_dir.display(drv_path).to_string(); let output_name_str: &str = output_name.as_ref(); let row: Option<(String,)> = sqlx::query_as( @@ -422,23 +421,18 @@ impl Connection { .fetch_optional(&mut *self.conn) .await?; - row.map(|(path,)| { - store_dir - .parse(&path) - .map_err(|e| sqlx::Error::Decode(Box::new(e))) - }) - .transpose() + row.map(|(path,)| Ok(store_dir.parse(&path)?)).transpose() } } impl Transaction<'_> { #[tracing::instrument(skip(self), err)] - pub async fn commit(self) -> sqlx::Result<()> { - self.tx.commit().await + pub async fn commit(self) -> crate::Result<()> { + Ok(self.tx.commit().await?) } #[tracing::instrument(skip(self, v), err)] - pub async fn update_build(&mut self, build_id: i32, v: UpdateBuild<'_>) -> sqlx::Result<()> { + pub async fn update_build(&mut self, build_id: i32, v: UpdateBuild<'_>) -> crate::Result<()> { sqlx::query!( r#" UPDATE builds SET @@ -475,7 +469,7 @@ impl Transaction<'_> { start_time: i32, stop_time: i32, is_cached_build: bool, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { sqlx::query!( r#" UPDATE builds SET @@ -503,7 +497,7 @@ impl Transaction<'_> { &mut self, build_id: i32, status: BuildStatus, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { #[allow(clippy::cast_possible_truncation)] sqlx::query!( r#" @@ -533,7 +527,7 @@ impl Transaction<'_> { build_id: i32, name: &str, path: &StorePath, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { let path = store_dir.display(path).to_string(); // TODO: support inserting multiple at the same time sqlx::query!( @@ -552,7 +546,7 @@ impl Transaction<'_> { &mut self, store_dir: &StoreDir, path: &StorePath, - ) -> sqlx::Result> { + ) -> crate::Result> { let path = store_dir.display(path).to_string(); Ok(sqlx::query!("SELECT MAX(build) FROM buildsteps WHERE drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1", path.as_str()) .fetch_optional(&mut *self.tx) @@ -565,7 +559,7 @@ impl Transaction<'_> { &mut self, store_dir: &StoreDir, path: &StorePath, - ) -> sqlx::Result> { + ) -> crate::Result> { let path = store_dir.display(path).to_string(); Ok(sqlx::query!( r#" @@ -589,7 +583,7 @@ impl Transaction<'_> { store_dir: &StoreDir, drv_path: &StorePath, name: &str, - ) -> sqlx::Result> { + ) -> crate::Result> { let drv_path = store_dir.display(drv_path).to_string(); Ok(sqlx::query!( r#" @@ -614,7 +608,7 @@ impl Transaction<'_> { &mut self, store_dir: &StoreDir, step: InsertBuildStep<'_>, - ) -> sqlx::Result> { + ) -> crate::Result> { let drv_path = store_dir.display(step.drv_path).to_string(); let success = sqlx::query!( r#" @@ -671,7 +665,7 @@ impl Transaction<'_> { &mut self, store_dir: &StoreDir, outputs: &[InsertBuildStepOutput], - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { if outputs.is_empty() { return Ok(()); } @@ -703,7 +697,7 @@ impl Transaction<'_> { step_nr: i32, name: &str, path: &StorePath, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { let path = store_dir.display(path).to_string(); // TODO: support inserting multiple at the same time sqlx::query!( @@ -723,7 +717,7 @@ impl Transaction<'_> { &mut self, store_dir: &StoreDir, drv_path: &StorePath, - ) -> sqlx::Result> { + ) -> crate::Result> { let drv_path = store_dir.display(drv_path).to_string(); let items: Vec<(String, String)> = sqlx::query_as( r"SELECT o.name, o.path @@ -737,22 +731,19 @@ impl Transaction<'_> { items .into_iter() - .map(|(name, path)| -> anyhow::Result<_> { - let name: OutputName = name.parse().context("invalid output name from DB")?; - let path: StorePath = store_dir - .parse(&path) - .context("invalid store path from DB")?; + .map(|(name, path)| -> crate::Result<_> { + let name: OutputName = name.parse()?; + let path: StorePath = store_dir.parse(&path)?; Ok((name, path)) }) - .collect::>() - .map_err(|e| sqlx::Error::Decode(e.into_boxed_dyn_error())) + .collect() } #[tracing::instrument(skip(self, res), err)] pub async fn update_build_step_in_finish( &mut self, res: UpdateBuildStepInFinish<'_>, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { sqlx::query!( r#" UPDATE buildsteps SET @@ -790,7 +781,7 @@ impl Transaction<'_> { store_dir: &StoreDir, build_id: i32, step_nr: i32, - ) -> sqlx::Result> { + ) -> crate::Result> { sqlx::query!( "SELECT drvPath FROM BuildSteps WHERE build = $1 AND stepnr = $2", build_id, @@ -799,16 +790,13 @@ impl Transaction<'_> { .fetch_optional(&mut *self.tx) .await? .and_then(|v| v.drvpath) - .map(|p| { - store_dir - .parse(&p) - .map_err(|e| sqlx::Error::Decode(Box::new(e))) - }) + .map(|p| store_dir.parse(&p)) .transpose() + .map_err(crate::Error::from) } #[tracing::instrument(skip(self, build_id), err)] - pub async fn check_if_build_is_not_finished(&mut self, build_id: i32) -> sqlx::Result { + pub async fn check_if_build_is_not_finished(&mut self, build_id: i32) -> crate::Result { Ok(sqlx::query!( "SELECT id FROM builds WHERE id = $1 AND finished = 0", build_id, @@ -819,7 +807,10 @@ impl Transaction<'_> { } #[tracing::instrument(skip(self, p), err)] - pub async fn insert_build_product(&mut self, p: InsertBuildProduct<'_>) -> sqlx::Result<()> { + pub(crate) async fn insert_build_product( + &mut self, + p: InsertBuildProduct<'_>, + ) -> crate::Result<()> { sqlx::query!( r#" INSERT INTO buildproducts ( @@ -841,7 +832,13 @@ impl Transaction<'_> { p.r#type, p.subtype, p.file_size, - p.sha256hash, + p.sha256hash.map(|h| { + let bytes: &[u8] = h.as_ref(); + bytes.iter().fold(String::new(), |mut output, b| { + let _ = write!(output, "{b:02x}"); + output + }) + }) as Option, p.path, p.name, p.default_path, @@ -852,7 +849,7 @@ impl Transaction<'_> { } #[tracing::instrument(skip(self, build_id), err)] - pub async fn delete_build_products_by_build_id(&mut self, build_id: i32) -> sqlx::Result<()> { + pub async fn delete_build_products_by_build_id(&mut self, build_id: i32) -> crate::Result<()> { sqlx::query!("DELETE FROM buildproducts WHERE build = $1", build_id) .execute(&mut *self.tx) .await?; @@ -860,7 +857,10 @@ impl Transaction<'_> { } #[tracing::instrument(skip(self, metric), err)] - pub async fn insert_build_metric(&mut self, metric: InsertBuildMetric<'_>) -> sqlx::Result<()> { + pub(crate) async fn insert_build_metric( + &mut self, + metric: InsertBuildMetric<'_>, + ) -> crate::Result<()> { sqlx::query!( r#" INSERT INTO buildmetrics ( @@ -891,7 +891,7 @@ impl Transaction<'_> { } #[tracing::instrument(skip(self, build_id), err)] - pub async fn delete_build_metrics_by_build_id(&mut self, build_id: i32) -> sqlx::Result<()> { + pub async fn delete_build_metrics_by_build_id(&mut self, build_id: i32) -> crate::Result<()> { sqlx::query!("DELETE FROM buildmetrics WHERE build = $1", build_id) .execute(&mut *self.tx) .await?; @@ -903,7 +903,7 @@ impl Transaction<'_> { &mut self, store_dir: &StoreDir, path: &StorePath, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { let path = store_dir.display(path).to_string(); sqlx::query!( r#" @@ -946,7 +946,7 @@ impl Transaction<'_> { error_msg: Option, propagated_from: Option, outputs: BTreeMap>, - ) -> sqlx::Result { + ) -> crate::Result { let step_nr = loop { if let Some(step_nr) = self .insert_build_step( @@ -1009,7 +1009,7 @@ impl Transaction<'_> { build_id: crate::models::BuildID, drv_path: &StorePath, outputs: BTreeMap, - ) -> anyhow::Result { + ) -> crate::Result { let step_nr = loop { if let Some(step_nr) = self .insert_build_step( @@ -1063,7 +1063,7 @@ impl Transaction<'_> { build_id: crate::models::BuildID, drv_path: &StorePath, output: (OutputName, Option), - ) -> anyhow::Result { + ) -> crate::Result { let step_nr = loop { if let Some(step_nr) = self .insert_build_step( @@ -1113,7 +1113,7 @@ impl Transaction<'_> { start_time: i32, stop_time: i32, store_dir: &StoreDir, - ) -> anyhow::Result<()> { + ) -> crate::Result<()> { if build.finished_in_db { return Ok(()); } @@ -1148,26 +1148,27 @@ impl Transaction<'_> { self.delete_build_products_by_build_id(build.id).await?; for (nr, p) in build.products.iter().enumerate() { + let path_str = p.path.print(store_dir); self.insert_build_product(InsertBuildProduct { build_id: build.id, product_nr: i32::try_from(nr + 1)?, - r#type: p.r#type, - subtype: p.subtype, - file_size: p.filesize, - sha256hash: p.sha256hash, - path: p.path.as_deref().unwrap_or_default(), - name: p.name, - default_path: p.defaultpath.unwrap_or_default(), + r#type: &p.r#type, + subtype: &p.subtype, + file_size: p.file_size.and_then(|s| i64::try_from(s).ok()), + sha256hash: p.sha256hash.as_ref(), + path: &path_str, + name: &p.name, + default_path: &p.default_path, }) .await?; } self.delete_build_metrics_by_build_id(build.id).await?; - for m in &build.metrics { + for (name, m) in &build.metrics { self.insert_build_metric(InsertBuildMetric { build_id: build.id, - name: m.name, - unit: m.unit, + name, + unit: m.unit.as_deref(), value: m.value, project: build.project_name, jobset: build.jobset_name, @@ -1182,7 +1183,7 @@ impl Transaction<'_> { impl Transaction<'_> { #[tracing::instrument(skip(self), err)] - async fn notify_any(&mut self, channel: &str, msg: &str) -> sqlx::Result<()> { + async fn notify_any(&mut self, channel: &str, msg: &str) -> crate::Result<()> { sqlx::query( r"SELECT pg_notify(chan, payload) from (values ($1, $2)) notifies(chan, payload)", ) @@ -1194,13 +1195,13 @@ impl Transaction<'_> { } #[tracing::instrument(skip(self), err)] - pub async fn notify_builds_added(&mut self) -> sqlx::Result<()> { + pub async fn notify_builds_added(&mut self) -> crate::Result<()> { self.notify_any("builds_added", "?").await?; Ok(()) } #[tracing::instrument(skip(self, build_id), err)] - pub async fn notify_build_started(&mut self, build_id: i32) -> sqlx::Result<()> { + pub async fn notify_build_started(&mut self, build_id: i32) -> crate::Result<()> { self.notify_any("build_started", &build_id.to_string()) .await?; Ok(()) @@ -1211,7 +1212,7 @@ impl Transaction<'_> { &mut self, build_id: i32, dependent_ids: &[i32], - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { let mut q = vec![build_id.to_string()]; q.extend(dependent_ids.iter().map(ToString::to_string)); @@ -1220,7 +1221,7 @@ impl Transaction<'_> { } #[tracing::instrument(skip(self, build_id, step_nr,), err)] - pub async fn notify_step_started(&mut self, build_id: i32, step_nr: i32) -> sqlx::Result<()> { + pub async fn notify_step_started(&mut self, build_id: i32, step_nr: i32) -> crate::Result<()> { self.notify_any("step_started", &format!("{build_id}\t{step_nr}")) .await?; Ok(()) @@ -1232,7 +1233,7 @@ impl Transaction<'_> { build_id: i32, step_nr: i32, log_file: &str, - ) -> sqlx::Result<()> { + ) -> crate::Result<()> { self.notify_any( "step_finished", &format!("{build_id}\t{step_nr}\t{log_file}"), diff --git a/subprojects/crates/db/src/error.rs b/subprojects/crates/db/src/error.rs new file mode 100644 index 000000000..d10314b6e --- /dev/null +++ b/subprojects/crates/db/src/error.rs @@ -0,0 +1,47 @@ +/// Data in the database couldn't be parsed into domain types. +#[derive(Debug, thiserror::Error)] +pub enum DataError { + #[error(transparent)] + StorePath(#[from] harmonia_store_path::ParseStorePathError), + + #[error(transparent)] + StorePathName(#[from] harmonia_store_path::StorePathNameError), + + #[error(transparent)] + IntConversion(#[from] std::num::TryFromIntError), + + #[error("build product #{productnr} for build {build_id} has no path")] + BuildProductMissingPath { build_id: i32, productnr: i32 }, +} + +/// Errors from the db crate. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Database connection or query error — infrastructure failure. + #[error(transparent)] + Sql(#[from] sqlx::Error), + + /// Data in the database couldn't be parsed into domain types. + #[error("invalid data from database: {0}")] + Data(#[from] DataError), +} + +impl From for Error { + fn from(e: harmonia_store_path::ParseStorePathError) -> Self { + Self::Data(e.into()) + } +} + +impl From for Error { + fn from(e: harmonia_store_path::StorePathNameError) -> Self { + Self::Data(e.into()) + } +} + +impl From for Error { + fn from(e: std::num::TryFromIntError) -> Self { + Self::Data(e.into()) + } +} + +pub type Result = std::result::Result; diff --git a/subprojects/crates/db/src/lib.rs b/subprojects/crates/db/src/lib.rs index 7bac16acd..c6462822d 100644 --- a/subprojects/crates/db/src/lib.rs +++ b/subprojects/crates/db/src/lib.rs @@ -14,13 +14,14 @@ #![allow(clippy::missing_errors_doc)] mod connection; +mod error; pub mod models; use std::str::FromStr as _; pub use connection::{Connection, Transaction}; -pub use harmonia_store_core::store_path::StoreDir; -pub use sqlx::Error; +pub use error::{DataError, Error, Result}; +pub use harmonia_store_path::StoreDir; #[derive(Debug, Clone)] pub struct Database { @@ -28,7 +29,7 @@ pub struct Database { } impl Database { - pub async fn new(url: &str, max_connections: u32) -> Result { + pub async fn new(url: &str, max_connections: u32) -> Result { Ok(Self { pool: sqlx::postgres::PgPoolOptions::new() .max_connections(max_connections) @@ -37,13 +38,13 @@ impl Database { }) } - pub async fn get(&self) -> Result { + pub async fn get(&self) -> Result { let conn = self.pool.acquire().await?; Ok(Connection::new(conn)) } #[tracing::instrument(skip(self, url), err)] - pub fn reconfigure_pool(&self, url: &str) -> anyhow::Result<()> { + pub fn reconfigure_pool(&self, url: &str) -> Result<()> { // TODO: ability to change max_connections by dropping the pool and recreating it self.pool .set_connect_options(sqlx::postgres::PgConnectOptions::from_str(url)?); @@ -54,8 +55,8 @@ impl Database { &self, channels: Vec<&str>, ) -> Result< - impl futures::Stream> + Unpin, - Error, + impl futures::Stream> + + Unpin, > { let mut listener = sqlx::postgres::PgListener::connect_with(&self.pool).await?; listener.listen_all(channels).await?; diff --git a/subprojects/crates/db/src/models.rs b/subprojects/crates/db/src/models.rs index e289529e4..1a3f7de37 100644 --- a/subprojects/crates/db/src/models.rs +++ b/subprojects/crates/db/src/models.rs @@ -1,5 +1,7 @@ -use harmonia_store_core::derived_path::OutputName; -use harmonia_store_core::store_path::{ParseStorePathError, StoreDir, StorePath}; +use std::collections::BTreeMap; + +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_path::{ParseStorePathError, StoreDir, StorePath}; use hashbrown::HashMap; pub type BuildID = i32; @@ -77,7 +79,7 @@ pub struct BuildSmall { } #[derive(Debug)] -pub struct Build { +pub struct Build { pub id: BuildID, pub jobset_id: i32, pub project: String, @@ -151,7 +153,7 @@ pub struct InsertBuildStep<'a> { } #[derive(Debug)] -pub struct InsertBuildStepOutput { +pub struct InsertBuildStepOutput { pub build_id: BuildID, pub step_nr: i32, pub name: OutputName, @@ -194,20 +196,20 @@ pub struct UpdateBuildStepInFinish<'a> { } #[derive(Debug)] -pub struct InsertBuildProduct<'a> { +pub(crate) struct InsertBuildProduct<'a> { pub build_id: BuildID, pub product_nr: i32, pub r#type: &'a str, pub subtype: &'a str, pub file_size: Option, - pub sha256hash: Option<&'a str>, + pub sha256hash: Option<&'a harmonia_utils_hash::Sha256>, pub path: &'a str, pub name: &'a str, pub default_path: &'a str, } #[derive(Debug)] -pub struct InsertBuildMetric<'a> { +pub(crate) struct InsertBuildMetric<'a> { pub build_id: BuildID, pub name: &'a str, pub unit: Option<&'a str>, @@ -227,61 +229,83 @@ pub struct BuildOutput { pub size: Option, } +/// A build product row from the `buildproducts` table. +/// +/// `buildproducts.path` is a filesystem path that may include a sub-path below +/// a store output (e.g. `doc manual $doc/share/doc/nix/manual index.html`). +/// The type parameter `Path` controls how that column is represented: +/// +/// Raw DB row for build products. Column names match the SQL schema. +/// Use [`BuildProductRow::into_build_product`] to convert to the typed +/// [`nix_support::BuildProduct`]. #[derive(Debug)] -pub struct OwnedBuildProduct { +pub(crate) struct BuildProductRow { + pub build: i32, + pub productnr: i32, pub r#type: String, pub subtype: String, pub filesize: Option, pub sha256hash: Option, - pub path: Option, + pub path: Option, pub name: String, pub defaultpath: Option, } -impl OwnedBuildProduct { - pub fn parse_paths( +impl BuildProductRow { + pub(crate) fn into_build_product( self, store_dir: &StoreDir, - ) -> Result { - Ok(OwnedBuildProduct { + ) -> Result { + let path_str = self.path.ok_or(crate::DataError::BuildProductMissingPath { + build_id: self.build, + productnr: self.productnr, + })?; + let path = store_path_utils::RelativeStorePath::from_path(store_dir, &path_str)?; + let sha256hash = self.sha256hash.and_then(|s| { + let mut bytes = [0u8; 32]; + if s.len() != 64 { + return None; + } + for (i, chunk) in s.as_bytes().chunks(2).enumerate() { + bytes[i] = u8::from_str_radix(std::str::from_utf8(chunk).ok()?, 16).ok()?; + } + harmonia_utils_hash::Sha256::from_slice(&bytes).ok() + }); + Ok(nix_support::BuildProduct { + path, + default_path: self.defaultpath.unwrap_or_default(), r#type: self.r#type, subtype: self.subtype, - filesize: self.filesize, - sha256hash: self.sha256hash, - path: self.path.map(|p| store_dir.parse(&p)).transpose()?, name: self.name, - defaultpath: self.defaultpath, + is_regular: self.filesize.is_some(), + #[allow(clippy::cast_sign_loss)] + file_size: self.filesize.map(|v| v as u64), + sha256hash, }) } } #[derive(Debug)] -pub struct BuildProduct<'a> { - pub r#type: &'a str, - pub subtype: &'a str, - pub filesize: Option, - pub sha256hash: Option<&'a str>, - pub path: Option, - pub name: &'a str, - pub defaultpath: Option<&'a str>, -} - -#[derive(Debug)] -pub struct OwnedBuildMetric { +pub(crate) struct OwnedBuildMetric { pub name: String, pub unit: Option, pub value: f64, } -#[derive(Debug)] -pub struct BuildMetric<'a> { - pub name: &'a str, - pub unit: Option<&'a str>, - pub value: f64, +impl From for (nix_support::BuildMetricName, nix_support::BuildMetric) { + fn from(m: OwnedBuildMetric) -> Self { + ( + m.name, + nix_support::BuildMetric { + unit: m.unit, + value: m.value, + }, + ) + } } #[derive(Debug)] -pub struct MarkBuildSuccessData<'a, StorePath = harmonia_store_core::store_path::StorePath> { +pub struct MarkBuildSuccessData<'a, StorePath = harmonia_store_path::StorePath> { pub id: BuildID, pub name: &'a str, pub project_name: &'a str, @@ -294,6 +318,98 @@ pub struct MarkBuildSuccessData<'a, StorePath = harmonia_store_core::store_path: pub size: u64, pub release_name: Option<&'a str>, pub outputs: HashMap, - pub products: Vec>, - pub metrics: Vec>, + pub products: Vec, + pub metrics: BTreeMap, +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn make_row(path: Option<&str>) -> BuildProductRow { + BuildProductRow { + build: 1, + productnr: 1, + r#type: "doc".into(), + subtype: "manual".into(), + filesize: None, + sha256hash: None, + path: path.map(Into::into), + name: "test-product".into(), + defaultpath: Some("index.html".into()), + } + } + + #[test] + fn into_build_product_subpath() { + let store_dir = StoreDir::default(); + let bp = make_row(Some( + "/nix/store/bwqqp42xqn37z31dapi7jrhy8iwc2zsx-nix-manual-2.31.4/share/doc/nix/manual", + )) + .into_build_product(&store_dir) + .unwrap(); + + assert_eq!( + bp.path.base_path.to_string(), + "bwqqp42xqn37z31dapi7jrhy8iwc2zsx-nix-manual-2.31.4" + ); + assert_eq!(&*bp.path.relative_path, "share/doc/nix/manual"); + } + + #[test] + fn into_build_product_bare_store_path() { + let store_dir = StoreDir::default(); + let bp = make_row(Some( + "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-example-1.0", + )) + .into_build_product(&store_dir) + .unwrap(); + + assert_eq!( + bp.path.base_path.to_string(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-example-1.0" + ); + assert!(bp.path.relative_path.is_empty()); + } + + #[test] + fn into_build_product_no_path_errors() { + let store_dir = StoreDir::default(); + let result = make_row(None).into_build_product(&store_dir); + assert!(result.is_err()); + } + + #[test] + fn into_build_product_sha256_roundtrip() { + let store_dir = StoreDir::default(); + let bp = BuildProductRow { + sha256hash: Some( + "4306152c73d2a7a01dbac16ba48f45fa4ae5b746a1d282638524ae2ae93af210".into(), + ), + filesize: Some(12345), + ..make_row(Some( + "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-example-1.0", + )) + } + .into_build_product(&store_dir) + .unwrap(); + + assert!(bp.sha256hash.is_some()); + assert_eq!(bp.file_size, Some(12345)); + assert!(bp.is_regular); + } + + #[test] + fn build_metric_from_db() { + let owned = OwnedBuildMetric { + name: "closureSize".into(), + unit: Some("bytes".into()), + value: 145_623_040.0, + }; + let (name, metric): (nix_support::BuildMetricName, nix_support::BuildMetric) = owned.into(); + assert_eq!(name, "closureSize"); + assert_eq!(metric.unit, Some("bytes".into())); + assert!((metric.value - 145_623_040.0).abs() < f64::EPSILON); + } } diff --git a/subprojects/crates/shared/Cargo.toml b/subprojects/crates/nix-support/Cargo.toml similarity index 57% rename from subprojects/crates/shared/Cargo.toml rename to subprojects/crates/nix-support/Cargo.toml index e4dbdc742..34fde524a 100644 --- a/subprojects/crates/shared/Cargo.toml +++ b/subprojects/crates/nix-support/Cargo.toml @@ -1,17 +1,18 @@ [package] -name = "shared" -version = "0.1.0" +name = "nix-support" +version.workspace = true edition = "2024" license = "GPL-3.0" rust-version.workspace = true [dependencies] -anyhow.workspace = true fs-err = { workspace = true, features = [ "tokio" ] } -prost.workspace = true regex.workspace = true sha2.workspace = true tokio = { workspace = true, features = [ "full" ] } tracing.workspace = true -nix-utils = { path = "../nix-utils" } +harmonia-store-derivation.workspace = true +harmonia-store-path.workspace = true +harmonia-utils-hash.workspace = true +store-path-utils.workspace = true diff --git a/subprojects/crates/nix-support/src/lib.rs b/subprojects/crates/nix-support/src/lib.rs new file mode 100644 index 000000000..a09544f88 --- /dev/null +++ b/subprojects/crates/nix-support/src/lib.rs @@ -0,0 +1,567 @@ +//! Parser for `nix-support/` files (build products, metrics, release name). +//! +//! This crate reads the `nix-support/hydra-build-products`, +//! `nix-support/hydra-metrics`, `nix-support/hydra-release-name`, and +//! `nix-support/failed` files from store path outputs, producing typed +//! [`NixSupport`] data. +//! +//! Paths are represented as [`StorePath`] or [`RelativeStorePath`] rather +//! than raw strings, so callers resolve to the real filesystem only at +//! the IO boundary. + +#![forbid(unsafe_code)] +#![deny( + clippy::all, + clippy::pedantic, + clippy::expect_used, + clippy::unwrap_used, + future_incompatible, + missing_debug_implementations, + nonstandard_style, + unreachable_pub, + missing_copy_implementations, + unused_qualifications +)] +#![allow(clippy::missing_errors_doc)] + +use std::{collections::BTreeMap, os::unix::fs::MetadataExt as _, sync::LazyLock}; + +use sha2::{Digest as _, Sha256}; +use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, BufReader}; + +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_path::StorePath; +use store_path_utils::RelativeStorePath; + +#[allow(clippy::expect_used)] +static VALIDATE_METRICS_NAME: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9._-]+").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static VALIDATE_METRICS_UNIT: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9._%-]+").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static VALIDATE_RELEASE_NAME: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9.@:_-]+").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static VALIDATE_PRODUCT_NAME: LazyLock = + LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9.@:_ -]*").expect("Failed to compile regex")); +#[allow(clippy::expect_used)] +static BUILD_PRODUCT_PARSER: LazyLock = LazyLock::new(|| { + regex::Regex::new( + r#"([a-zA-Z0-9_-]+)\s+([a-zA-Z0-9_-]+)\s+(\"[^\"]+\"|[^\"\s<>]+)(\s+([^\"\s<>]+))?"#, + ) + .expect("Failed to compile regex") +}); + +#[derive(Debug, Clone, PartialEq)] +pub struct BuildProduct { + pub path: RelativeStorePath, + pub default_path: String, + + pub r#type: String, + pub subtype: String, + pub name: String, + + pub is_regular: bool, + + pub sha256hash: Option, + pub file_size: Option, +} + +pub type BuildMetricName = String; + +#[derive(Debug, Clone, PartialEq)] +pub struct BuildMetric { + pub unit: Option, + pub value: f64, +} + +#[derive(Debug, Default, Clone, PartialEq)] +pub struct NixSupport { + pub failed: bool, + pub hydra_release_name: Option, + pub metrics: BTreeMap, + pub products: Vec, +} + +/// File metadata needed for build products. +#[derive(Debug, Clone, Copy)] +pub struct FileMetadata { + pub is_regular: bool, + pub size: u64, +} + +/// Abstraction over filesystem operations needed for parsing build products. +/// +/// The default implementation ([`FilesystemOperations`]) reads from the real +/// filesystem. Tests can provide a mock implementation. +pub trait FsOperations { + fn get_file_info(&self, path: &RelativeStorePath) + -> impl Future>; + + fn hash_file( + &self, + path: &RelativeStorePath, + ) -> impl Future>; +} + +/// Real filesystem implementation of [`FsOperations`]. +#[derive(Debug, Clone)] +pub struct FilesystemOperations { + pub real_store_dir: std::path::PathBuf, +} + +impl FilesystemOperations { + fn resolve(&self, path: &RelativeStorePath) -> std::path::PathBuf { + let mut p = self.real_store_dir.join(path.base_path.to_string()); + if !path.relative_path.is_empty() { + p = p.join(&*path.relative_path); + } + p + } +} + +impl FsOperations for FilesystemOperations { + async fn get_file_info(&self, path: &RelativeStorePath) -> Option { + let real = self.resolve(path); + let m = fs_err::tokio::metadata(&real).await.ok()?; + Some(FileMetadata { + is_regular: m.is_file(), + size: m.size(), + }) + } + + async fn hash_file(&self, path: &RelativeStorePath) -> Option { + let real = self.resolve(path); + let file = fs_err::tokio::File::open(&real).await.ok()?; + let mut reader = BufReader::new(file); + let mut hasher = Sha256::new(); + let mut buf = [0u8; 16 * 1024]; + loop { + let n = reader.read(&mut buf).await.ok()?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + let digest = hasher.finalize(); + harmonia_utils_hash::Sha256::from_slice(&digest).ok() + } +} + +fn parse_release_name(content: &str) -> Option { + let content = content.trim(); + if !content.is_empty() && VALIDATE_RELEASE_NAME.is_match(content) { + Some(content.to_owned()) + } else { + None + } +} + +fn parse_metric(line: &str) -> Option<(BuildMetricName, BuildMetric)> { + let fields: Vec<&str> = line.split_whitespace().collect(); + if fields.len() < 2 || !VALIDATE_METRICS_NAME.is_match(fields[0]) { + return None; + } + + let value: f64 = fields[1].parse().ok()?; + + let unit = if fields.len() >= 3 && VALIDATE_METRICS_UNIT.is_match(fields[2]) { + Some(fields[2].to_owned()) + } else { + None + }; + + Some((fields[0].to_owned(), BuildMetric { unit, value })) +} + +/// Resolve a store path to a filesystem path. +fn real_path(store_dir: &std::path::Path, path: &StorePath) -> std::path::PathBuf { + store_dir.join(path.to_string()) +} + +async fn parse_build_product( + store_dir: &harmonia_store_path::StoreDir, + fs: &F, + line: &str, +) -> Option { + let captures = BUILD_PRODUCT_PARSER.captures(line)?; + + let s = captures[3].to_string(); + let path_str = if s.starts_with('"') && s.ends_with('"') { + s[1..s.len() - 1].to_string() + } else { + s + }; + + if path_str.is_empty() || !path_str.starts_with('/') { + return None; + } + + // Parse as a RelativeStorePath (uses logical store dir from the file) + let relative = RelativeStorePath::from_path(store_dir, &path_str).ok()?; + + let file_info = fs.get_file_info(&relative).await?; + + let name = { + let name = if relative.relative_path.is_empty() { + String::new() + } else { + std::path::Path::new(&*relative.relative_path) + .file_name() + .and_then(|f| f.to_str()) + .map(ToOwned::to_owned) + .unwrap_or_default() + }; + if VALIDATE_PRODUCT_NAME.is_match(&name) { + name + } else { + String::new() + } + }; + + let sha256hash = if file_info.is_regular { + fs.hash_file(&relative).await + } else { + None + }; + + Some(BuildProduct { + r#type: captures[1].to_string(), + subtype: captures[2].to_string(), + path: relative, + default_path: captures + .get(5) + .map(|m| m.as_str().to_string()) + .unwrap_or_default(), + name, + is_regular: file_info.is_regular, + file_size: if file_info.is_regular { + Some(file_info.size) + } else { + None + }, + sha256hash, + }) +} + +impl NixSupport { + /// Monoidal combine: merge another `NixSupport` into this one. + /// + /// - `failed`: OR (any output failed → whole build failed) + /// - `hydra_release_name`: last wins + /// - `metrics`: last wins per name + /// - `products`: append + pub fn combine(&mut self, other: Self) { + self.failed |= other.failed; + if other.hydra_release_name.is_some() { + self.hydra_release_name = other.hydra_release_name; + } + self.metrics.extend(other.metrics); + self.products.extend(other.products); + } +} + +/// Parse `nix-support/` files for a single output. +/// +/// `store_dir` is the logical store directory (e.g. `/nix/store`), used to +/// parse paths found inside `hydra-build-products` files. +/// +/// `real_store_dir` is where the store objects actually live on the filesystem. +/// +/// `fs` provides filesystem access for build product metadata and hashing +/// (see [`FilesystemOperations`] for the real implementation). +pub async fn parse_nix_support_for_output( + store_dir: &harmonia_store_path::StoreDir, + real_store_dir: &std::path::Path, + fs: &F, + output_name: &OutputName, + output: &StorePath, +) -> std::io::Result { + let output_full_path = real_path(real_store_dir, output); + + let mut metrics = BTreeMap::new(); + let file_path = output_full_path.join("nix-support/hydra-metrics"); + if let Ok(file) = fs_err::tokio::File::open(&file_path).await { + let reader = BufReader::new(file); + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + if let Some((name, m)) = parse_metric(&line) { + metrics.insert(name, m); + } + } + } + + let failed = fs_err::tokio::try_exists(output_full_path.join("nix-support/failed")) + .await + .unwrap_or_default(); + + let hydra_release_name = if let Ok(v) = + fs_err::tokio::read_to_string(output_full_path.join("nix-support/hydra-release-name")).await + { + parse_release_name(&v) + } else { + None + }; + + let mut products = Vec::new(); + let products_path = output_full_path.join("nix-support/hydra-build-products"); + if let Ok(file) = fs_err::tokio::File::open(&products_path).await { + let reader = BufReader::new(file); + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + if let Some(o) = Box::pin(parse_build_product(store_dir, fs, &line)).await { + products.push(o); + } + } + } else { + // No explicit products — add the output itself as a "nix-build" product + let output_rel = RelativeStorePath { + base_path: output.clone(), + relative_path: "".into(), + }; + if let Some(info) = fs.get_file_info(&output_rel).await + && !info.is_regular + { + products.push(BuildProduct { + r#type: "nix-build".to_string(), + subtype: if output_name.as_ref() == "out" { + String::new() + } else { + output_name.to_string() + }, + path: output_rel, + name: output.name().to_string(), + default_path: String::new(), + is_regular: false, + file_size: None, + sha256hash: None, + }); + } + } + + Ok(NixSupport { + failed, + hydra_release_name, + metrics, + products, + }) +} + +/// Parse `nix-support/` files from all outputs, returning per-output data. +pub async fn parse_nix_support_from_outputs( + store_dir: &harmonia_store_path::StoreDir, + real_store_dir: &std::path::Path, + fs: &F, + derivation_outputs: &BTreeMap, +) -> std::io::Result> { + let mut result = BTreeMap::new(); + for (name, path) in derivation_outputs { + let ns = parse_nix_support_for_output(store_dir, real_store_dir, fs, name, path).await?; + result.insert(name.clone(), ns); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + + #[derive(Debug, Clone)] + struct DummyFsOperations { + valid_file: bool, + metadata: FileMetadata, + file_hash: Option, + } + + impl FsOperations for DummyFsOperations { + async fn get_file_info(&self, _: &RelativeStorePath) -> Option { + if self.valid_file { + Some(self.metadata) + } else { + None + } + } + + async fn hash_file(&self, _: &RelativeStorePath) -> Option { + self.file_hash + } + } + + #[tokio::test] + async fn test_build_product_with_mock() { + let store_dir = StoreDir::default(); + let output: StorePath = "ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso" + .parse() + .unwrap(); + let line = format!("file iso /nix/store/{output}/iso/custom.iso"); + let fs = DummyFsOperations { + valid_file: true, + metadata: FileMetadata { + is_regular: true, + size: 12345, + }, + file_hash: Some( + harmonia_utils_hash::Sha256::from_slice(&[ + 0x43, 0x06, 0x15, 0x2c, 0x73, 0xd2, 0xa7, 0xa0, 0x1d, 0xba, 0xc1, 0x6b, 0xa4, + 0x8f, 0x45, 0xfa, 0x4a, 0xe5, 0xb7, 0x46, 0xa1, 0xd2, 0x82, 0x63, 0x85, 0x24, + 0xae, 0x2a, 0xe9, 0x3a, 0xf2, 0x10, + ]) + .unwrap(), + ), + }; + let bp = Box::pin(parse_build_product(&store_dir, &fs, &line)) + .await + .unwrap(); + assert!(bp.is_regular); + assert_eq!(bp.name, "custom.iso"); + assert_eq!(bp.r#type, "file"); + assert_eq!(bp.subtype, "iso"); + assert_eq!(bp.file_size, Some(12345)); + assert!(bp.sha256hash.is_some()); + assert_eq!(bp.path.base_path, output); + assert_eq!(&*bp.path.relative_path, "iso/custom.iso"); + } + + #[tokio::test] + async fn test_build_product_invalid_file_returns_none() { + let store_dir = StoreDir::default(); + let line = "file iso /nix/store/ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso/iso/custom.iso"; + let fs = DummyFsOperations { + valid_file: false, + metadata: FileMetadata { + is_regular: false, + size: 0, + }, + file_hash: None, + }; + let bp = Box::pin(parse_build_product(&store_dir, &fs, line)).await; + assert!(bp.is_none()); + } + + #[test] + fn test_parse_invalid_metric() { + let m = parse_metric("nix-env.qaCount"); + assert!(m.is_none()); + } + + #[test] + fn test_parse_metric_without_unit() { + let (name, m) = parse_metric("nix-env.qaCount 4").unwrap(); + assert_eq!(name, "nix-env.qaCount"); + assert!((m.value - 4.0_f64).abs() < f64::EPSILON); + assert_eq!(m.unit, None); + } + + #[test] + fn test_parse_metric_with_unit() { + let (name, m) = parse_metric("xzy.time 123.321 s").unwrap(); + assert_eq!(name, "xzy.time"); + assert!((m.value - 123.321_f64).abs() < f64::EPSILON); + assert_eq!(m.unit, Some("s".into())); + } + + #[test] + fn test_parse_metric_bad_value_skipped() { + let m = parse_metric("nix-env.qaCount notanumber"); + assert!(m.is_none()); + } + + #[test] + fn test_parse_release_name() { + let o = parse_release_name("nixos-25.11pre708350"); + assert_eq!(o, Some("nixos-25.11pre708350".into())); + } + + /// Create a fake store dir with a file at the given sub-path, + /// returning the store dir path. + async fn setup_fake_store( + store_path_base: &str, + sub_path: &str, + contents: &[u8], + ) -> std::path::PathBuf { + let store_dir = + std::env::temp_dir().join(format!("nix-support-test-{}", std::process::id())); + let full_dir = store_dir.join(store_path_base); + let file_path = full_dir.join(sub_path); + fs_err::tokio::create_dir_all(file_path.parent().unwrap()) + .await + .unwrap(); + fs_err::tokio::write(&file_path, contents).await.unwrap(); + store_dir + } + + use harmonia_store_path::StoreDir; + + #[tokio::test] + async fn test_build_product_regular_file() { + let store_path_base = "ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"; + let real_dir = setup_fake_store(store_path_base, "iso/custom.iso", b"fake iso data").await; + + // The hydra-build-products file uses the logical store dir + let store_dir = StoreDir::new("/nix/store").unwrap(); + let line = format!("file iso /nix/store/{store_path_base}/iso/custom.iso"); + + let bp = Box::pin(parse_build_product( + &store_dir, + &FilesystemOperations { + real_store_dir: real_dir.clone(), + }, + &line, + )) + .await + .unwrap(); + + let expected_output: StorePath = store_path_base.parse().unwrap(); + assert!(bp.is_regular); + assert_eq!(bp.name, "custom.iso"); + assert_eq!(bp.r#type, "file"); + assert_eq!(bp.subtype, "iso"); + assert_eq!(bp.file_size, Some(13)); + assert!(bp.sha256hash.is_some()); + assert_eq!(bp.path.base_path, expected_output); + assert_eq!(&*bp.path.relative_path, "iso/custom.iso"); + + fs_err::tokio::remove_dir_all(&real_dir).await.unwrap(); + } + + #[tokio::test] + async fn test_build_product_rejects_outside_store() { + let store_dir = StoreDir::default(); + let line = "file iso /tmp/evil/custom.iso"; + let fs = FilesystemOperations { + real_store_dir: store_dir.to_path().to_owned(), + }; + let bp = Box::pin(parse_build_product(&store_dir, &fs, line)).await; + assert!(bp.is_none()); + } + + #[tokio::test] + async fn test_build_product_output_path_has_empty_name() { + let store_path_base = "ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-test-1.0"; + let real_dir = + std::env::temp_dir().join(format!("nix-support-test-bare-{}", std::process::id())); + let output_file = real_dir.join(store_path_base); + fs_err::tokio::create_dir_all(&real_dir).await.unwrap(); + fs_err::tokio::write(&output_file, b"data").await.unwrap(); + + let store_dir = StoreDir::new("/nix/store").unwrap(); + let line = format!("file binary /nix/store/{store_path_base}"); + let bp = Box::pin(parse_build_product( + &store_dir, + &FilesystemOperations { + real_store_dir: real_dir.clone(), + }, + &line, + )) + .await + .unwrap(); + + // When the product path equals the output path, name should be empty + assert_eq!(bp.name, ""); + + fs_err::tokio::remove_dir_all(&real_dir).await.unwrap(); + } +} diff --git a/subprojects/crates/nix-utils/Cargo.toml b/subprojects/crates/nix-utils/Cargo.toml deleted file mode 100644 index 1816a2e3d..000000000 --- a/subprojects/crates/nix-utils/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "nix-utils" -version = "0.1.0" -edition = "2024" -license = "LGPL-2.1-only" -rust-version.workspace = true - -[dependencies] -anyhow.workspace = true -bytes.workspace = true -fs-err = { workspace = true, features = [ "tokio" ] } -futures.workspace = true -hashbrown.workspace = true -serde = { workspace = true, features = [ "derive" ] } -serde_json.workspace = true -smallvec.workspace = true -thiserror.workspace = true -tokio = { workspace = true, features = [ "full" ] } -tokio-stream = { workspace = true, features = [ "io-util" ] } -tokio-util = { workspace = true, features = [ "io", "io-util" ] } -tracing.workspace = true -url.workspace = true - -cxx.workspace = true -harmonia-store-aterm.workspace = true -harmonia-store-core.workspace = true -harmonia-utils-hash.workspace = true - -[build-dependencies] -cxx-build.workspace = true -pkg-config.workspace = true diff --git a/subprojects/crates/nix-utils/LICENSE b/subprojects/crates/nix-utils/LICENSE deleted file mode 100644 index 5ab7695ab..000000000 --- a/subprojects/crates/nix-utils/LICENSE +++ /dev/null @@ -1,504 +0,0 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts - as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.] - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some -specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You -can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better -strategy to use in any particular case, based on the explanations below. - - When we speak of free software, we are referring to freedom of use, -not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge -for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of -it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid -distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for -you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them -with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the -library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that -there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know -that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be -introduced by others. - - Finally, software patents pose a constant threat to the existence of -any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a -restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be -consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the -ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and -is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those -libraries into non-free programs. - - When a program is linked with a library, whether statically or using -a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary -General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General -Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it -does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less -of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many -libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to -encourage the widest possible use of a certain library, so that it becomes -a de-facto standard. To achieve this, non-free programs must be -allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this -case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free -programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in -non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating -system. - - Although the Lesser General Public License is Less protective of the -users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run -that program using a modified version of the Library. - - The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, whereas the latter must -be combined with the library in order to run. - - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other -program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of -this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. - - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also combine or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, - rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if - the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties with -this License. - - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add -an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus -excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random Hacker. - - , 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! - - diff --git a/subprojects/crates/nix-utils/build.rs b/subprojects/crates/nix-utils/build.rs deleted file mode 100644 index 3e0b0cc45..000000000 --- a/subprojects/crates/nix-utils/build.rs +++ /dev/null @@ -1,33 +0,0 @@ -fn main() { - if std::env::var("DOCS_RS").is_ok() { - return; - } - - println!("cargo:rerun-if-changed=include/"); - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=src/nix.cpp"); - println!("cargo:rerun-if-changed=src/lib.rs"); - println!("cargo:rerun-if-changed=src/cxx/"); - - let nix_main = pkg_config::probe_library("nix-main").unwrap(); - let nix_store = pkg_config::probe_library("nix-store").unwrap(); - let nix_util = pkg_config::probe_library("nix-util").unwrap(); - - cxx_build::bridges(["src/lib.rs"]) - .files(["src/nix.cpp", "src/cxx/utils.cpp"]) - .flag("-std=c++23") - .flag("-O2") - .includes(&nix_main.include_paths) - .compile("nix_utils"); - - // Re-emit link directives after compile() so that the nix shared libs - // appear after the static CXX bridge lib in the link order. - for lib in [&nix_main, &nix_store, &nix_util] { - for link_path in &lib.link_paths { - println!("cargo:rustc-link-search=native={}", link_path.display()); - } - for lib_name in &lib.libs { - println!("cargo:rustc-link-lib={}", lib_name); - } - } -} diff --git a/subprojects/crates/nix-utils/examples/copy_path.rs b/subprojects/crates/nix-utils/examples/copy_path.rs deleted file mode 100644 index cf3cab4a0..000000000 --- a/subprojects/crates/nix-utils/examples/copy_path.rs +++ /dev/null @@ -1,24 +0,0 @@ -use nix_utils::{self, copy_paths}; - -// requires env vars: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY - -#[tokio::main] -async fn main() { - let local = nix_utils::LocalStore::init(); - let remote = - nix_utils::RemoteStore::init("s3://store?region=unknown&endpoint=http://localhost:9000"); - nix_utils::set_verbosity(1); - - let res = copy_paths( - local.as_base_store(), - remote.as_base_store(), - &[nix_utils::parse_store_path( - "1r5zv195y7b7b5q2daf5p82s2m6r4rg4-CVE-2024-56406.patch", - )], - false, - false, - false, - ) - .await; - println!("copy res={res:?}"); -} diff --git a/subprojects/crates/nix-utils/examples/drv_parse.rs b/subprojects/crates/nix-utils/examples/drv_parse.rs deleted file mode 100644 index b3304524e..000000000 --- a/subprojects/crates/nix-utils/examples/drv_parse.rs +++ /dev/null @@ -1,12 +0,0 @@ -#[tokio::main] -async fn main() { - let store = nix_utils::LocalStore::init(); - let drv = nix_utils::query_drv( - &store, - &nix_utils::parse_store_path("5g60vyp4cbgwl12pav5apyi571smp62s-hello-2.12.2.drv"), - ) - .await - .unwrap(); - - println!("{drv:?}"); -} diff --git a/subprojects/crates/nix-utils/examples/export_file.rs b/subprojects/crates/nix-utils/examples/export_file.rs deleted file mode 100644 index b918fe9a2..000000000 --- a/subprojects/crates/nix-utils/examples/export_file.rs +++ /dev/null @@ -1,34 +0,0 @@ -use nix_utils::{self, BaseStore as _}; - -#[tokio::main] -async fn main() { - let store = nix_utils::LocalStore::init(); - - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::>(); - let closure = move |data: &[u8]| { - let data = Vec::from(data); - tx.send(data).is_ok() - }; - - let x = tokio::spawn(async move { - while let Some(x) = rx.recv().await { - print!("{}", String::from_utf8_lossy(&x)); - } - }); - - tokio::task::spawn_blocking(move || async move { - store - .export_paths( - &[nix_utils::parse_store_path( - "5g60vyp4cbgwl12pav5apyi571smp62s-hello-2.12.2.drv", - )], - closure, - ) - .unwrap(); - }) - .await - .unwrap() - .await; - - x.await.unwrap(); -} diff --git a/subprojects/crates/nix-utils/examples/get_settings.rs b/subprojects/crates/nix-utils/examples/get_settings.rs deleted file mode 100644 index 7813c105c..000000000 --- a/subprojects/crates/nix-utils/examples/get_settings.rs +++ /dev/null @@ -1,10 +0,0 @@ -fn main() { - let _store = nix_utils::LocalStore::init(); - println!("Store dir: {}", nix_utils::get_store_dir()); - println!("State dir: {}", nix_utils::get_state_dir()); - println!("System: {}", nix_utils::get_this_system()); - println!("Extra Platforms: {:?}", nix_utils::get_extra_platforms()); - println!("System features: {:?}", nix_utils::get_system_features()); - println!("Substituters: {:?}", nix_utils::get_substituters()); - println!("Use cgroups: {}", nix_utils::get_use_cgroups()); -} diff --git a/subprojects/crates/nix-utils/examples/import_file_fd.rs b/subprojects/crates/nix-utils/examples/import_file_fd.rs deleted file mode 100644 index 6cbe73081..000000000 --- a/subprojects/crates/nix-utils/examples/import_file_fd.rs +++ /dev/null @@ -1,35 +0,0 @@ -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - -use nix_utils::{self, BaseStore as _}; - -#[tokio::main] -async fn main() { - let store = nix_utils::LocalStore::init(); - - let file = fs_err::tokio::File::open("/tmp/test.nar").await.unwrap(); - let mut reader = tokio::io::BufReader::new(file); - - println!("Importing test.nar == 5g60vyp4cbgwl12pav5apyi571smp62s-hello-2.12.2.drv"); - let (mut rx, tx) = tokio::net::unix::pipe::pipe().unwrap(); - - tokio::spawn(async move { - let mut buf: [u8; 1] = [0; 1]; - loop { - let s = reader.read(&mut buf).await.unwrap(); - if s == 0 { - break; - } - let _ = rx.write(&buf).await.unwrap(); - } - let _ = rx.shutdown().await; - drop(rx); - }); - tokio::task::spawn_blocking(move || async move { - store - .import_paths_with_fd(tx.into_blocking_fd().unwrap(), false) - .unwrap(); - }) - .await - .unwrap() - .await; -} diff --git a/subprojects/crates/nix-utils/examples/import_file_stream.rs b/subprojects/crates/nix-utils/examples/import_file_stream.rs deleted file mode 100644 index 8e00c28a5..000000000 --- a/subprojects/crates/nix-utils/examples/import_file_stream.rs +++ /dev/null @@ -1,11 +0,0 @@ -use nix_utils::{self, BaseStore as _}; - -#[tokio::main] -async fn main() { - let store = nix_utils::LocalStore::init(); - - let file = fs_err::tokio::File::open("/tmp/test3.nar").await.unwrap(); - let stream = tokio_util::io::ReaderStream::new(file); - - store.import_paths(stream, false).await.unwrap(); -} diff --git a/subprojects/crates/nix-utils/examples/is_valid_path.rs b/subprojects/crates/nix-utils/examples/is_valid_path.rs deleted file mode 100644 index 1baf1e77d..000000000 --- a/subprojects/crates/nix-utils/examples/is_valid_path.rs +++ /dev/null @@ -1,13 +0,0 @@ -use nix_utils::{self, BaseStore as _}; - -#[tokio::main] -async fn main() { - let store = nix_utils::LocalStore::init(); - let store_dir = nix_utils::get_store_dir(); - println!( - "storepath={store_dir} valid={}", - store - .is_valid_path(&nix_utils::parse_store_path(store_dir.as_ref())) - .await - ); -} diff --git a/subprojects/crates/nix-utils/examples/list_nar.rs b/subprojects/crates/nix-utils/examples/list_nar.rs deleted file mode 100644 index 2502b9fec..000000000 --- a/subprojects/crates/nix-utils/examples/list_nar.rs +++ /dev/null @@ -1,14 +0,0 @@ -use nix_utils::BaseStore as _; - -#[tokio::main] -async fn main() { - let store = nix_utils::LocalStore::init(); - - let ls = store - .list_nar_deep(&nix_utils::parse_store_path( - "sqw9kyl8zrfnkklb3vp6gji9jw9qfgb5-hello-2.12.2", - )) - .await - .unwrap(); - println!("{ls:?}"); -} diff --git a/subprojects/crates/nix-utils/examples/path_infos.rs b/subprojects/crates/nix-utils/examples/path_infos.rs deleted file mode 100644 index ea2623674..000000000 --- a/subprojects/crates/nix-utils/examples/path_infos.rs +++ /dev/null @@ -1,28 +0,0 @@ -use nix_utils::BaseStore as _; - -#[tokio::main] -async fn main() { - let local = nix_utils::LocalStore::init(); - - let p1 = nix_utils::parse_store_path("ihl4ya67glh9815v1lanyqph0p7hdzfb-hdf5-cpp-1.14.6-bin"); - let p2 = nix_utils::parse_store_path("sgv5w811jvvxpjgmyw1n6l8hwfilha7x-hdf5-cpp-1.14.6-dev"); - let p3 = nix_utils::parse_store_path("vb6yrzk31ng8s6nzs4y4jq6qsjab3gxv-hdf5-cpp-1.14.6"); - - let infos = local.query_path_infos(&[&p1, &p2, &p3]).await; - - println!("{infos:?}"); - println!( - "closure_size {p1}: {}", - local.compute_closure_size(&p1).await - ); - println!( - "closure_size {p2}: {}", - local.compute_closure_size(&p2).await - ); - println!( - "closure_size {p3}: {}", - local.compute_closure_size(&p3).await - ); - - println!("stats: {:?}", local.get_store_stats()); -} diff --git a/subprojects/crates/nix-utils/examples/query_requisites.rs b/subprojects/crates/nix-utils/examples/query_requisites.rs deleted file mode 100644 index d2e64efa8..000000000 --- a/subprojects/crates/nix-utils/examples/query_requisites.rs +++ /dev/null @@ -1,20 +0,0 @@ -use nix_utils::BaseStore as _; - -#[tokio::main] -async fn main() { - let store = nix_utils::LocalStore::init(); - - let drv = nix_utils::parse_store_path("z3d15qi11dvljq5qz84kak3h0nb12wca-rsyslog-8.2510.0"); - let ps = store.query_requisites(&[&drv], false).await.unwrap(); - for p in ps { - println!("{}", store.print_store_path(&p)); - } - - println!(); - println!(); - - let ps = store.query_requisites(&[&drv], true).await.unwrap(); - for p in ps { - println!("{}", store.print_store_path(&p)); - } -} diff --git a/subprojects/crates/nix-utils/examples/stream_test.rs b/subprojects/crates/nix-utils/examples/stream_test.rs deleted file mode 100644 index 07d6c6e50..000000000 --- a/subprojects/crates/nix-utils/examples/stream_test.rs +++ /dev/null @@ -1,29 +0,0 @@ -use bytes::Bytes; -use tokio::io::AsyncReadExt; -use tokio_util::io::StreamReader; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Create a stream from an iterator. - let stream = tokio_stream::iter(vec![ - tokio::io::Result::Ok(Bytes::from_static(&[0, 1, 2, 3])), - tokio::io::Result::Ok(Bytes::from_static(&[4, 5, 6, 7])), - tokio::io::Result::Ok(Bytes::from_static(&[8, 9, 10, 11])), - ]); - - // Convert it to an AsyncRead. - let mut read = StreamReader::new(stream); - - // Read five bytes from the stream. - let mut buf = [0; 2]; - - loop { - let read = read.read(&mut buf).await?; - if read == 0 { - break; - } - println!("{buf:?}"); - } - - Ok(()) -} diff --git a/subprojects/crates/nix-utils/examples/upsert_file.rs b/subprojects/crates/nix-utils/examples/upsert_file.rs deleted file mode 100644 index c332e17be..000000000 --- a/subprojects/crates/nix-utils/examples/upsert_file.rs +++ /dev/null @@ -1,19 +0,0 @@ -// requires env vars: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY - -#[tokio::main] -async fn main() { - let store = - nix_utils::RemoteStore::init("s3://store?region=unknown&endpoint=http://localhost:9000"); - nix_utils::set_verbosity(1); - let res = store - .upsert_file( - "log/z4zxibgvmk4ikarbbpwjql21wjmdvy85-dbus-1.drv".to_string(), - std::path::PathBuf::from( - concat!(env!("CARGO_MANIFEST_DIR"), "/examples/upsert_file.rs").to_string(), - ), - "text/plain; charset=utf-8", - ) - .await; - - println!("upsert res={res:?}",); -} diff --git a/subprojects/crates/nix-utils/include/nix.h b/subprojects/crates/nix-utils/include/nix.h deleted file mode 100644 index 438508b81..000000000 --- a/subprojects/crates/nix-utils/include/nix.h +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once - -#include "rust/cxx.h" -#include -#include - -namespace nix_utils { -class StoreWrapper { -public: - StoreWrapper(nix::ref _store); - - nix::ref _store; -}; -} // namespace nix_utils - -// we need to include this after StoreWrapper -#include "nix-utils/src/lib.rs.h" - -namespace nix_utils { -void init_nix(); -std::unique_ptr init(rust::Str uri); - -rust::String get_store_dir(); -rust::String get_store_dir_for(const StoreWrapper &wrapper); -rust::String get_build_dir(); -rust::String get_state_dir(); -rust::String get_nix_version(); -rust::String get_this_system(); -rust::Vec get_extra_platforms(); -rust::Vec get_system_features(); -rust::Vec get_substituters(); - -bool get_use_cgroups(); -void set_verbosity(int32_t level); - -bool is_valid_path(const StoreWrapper &wrapper, rust::Str path); -InternalPathInfo query_path_info(const StoreWrapper &wrapper, rust::Str path); -void clear_path_info_cache(const StoreWrapper &wrapper); -uint64_t compute_closure_size(const StoreWrapper &wrapper, rust::Str path); -rust::Vec compute_fs_closure(const StoreWrapper &wrapper, - rust::Str path, bool flip_direction, - bool include_outputs, - bool include_derivers); -rust::Vec -compute_fs_closures(const StoreWrapper &wrapper, - rust::Slice paths, bool flip_direction, - bool include_outputs, bool include_derivers, bool toposort); -void upsert_file(const StoreWrapper &wrapper, rust::Str path, rust::Str data, - rust::Str mime_type); -StoreStats get_store_stats(const StoreWrapper &wrapper); -void copy_paths(const StoreWrapper &src_store, const StoreWrapper &dst_store, - rust::Slice paths, bool repair, - bool check_sigs, bool substitute); - -void import_paths( - const StoreWrapper &wrapper, bool check_sigs, size_t runtime, size_t reader, - rust::Fn, size_t, size_t, size_t)> callback, - size_t user_data); -void import_paths_with_fd(const StoreWrapper &wrapper, bool check_sigs, - int32_t fd); -void export_paths(const StoreWrapper &src_store, - rust::Slice paths, - rust::Fn, size_t)> callback, - size_t userdata); -void nar_from_path(const StoreWrapper &src_store, rust::Str path, - rust::Fn, size_t)> callback, - size_t userdata); - -rust::String list_nar_deep(const StoreWrapper &wrapper, rust::Str path); - -void ensure_path(const StoreWrapper &wrapper, rust::Str path); -rust::String to_real_path(const StoreWrapper &wrapper, rust::Str path); -rust::String write_derivation(const StoreWrapper &wrapper, rust::Str json); -} // namespace nix_utils diff --git a/subprojects/crates/nix-utils/include/utils.h b/subprojects/crates/nix-utils/include/utils.h deleted file mode 100644 index 8669bce1e..000000000 --- a/subprojects/crates/nix-utils/include/utils.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "rust/cxx.h" -#include - -#define AS_VIEW(rstr) std::string_view(rstr.data(), rstr.length()) -#define AS_STRING(rstr) std::string(rstr.data(), rstr.length()) - -rust::String extract_opt_path(const nix::Store &store, - const std::optional &v); -rust::Vec extract_path_set(const nix::Store &store, - const nix::StorePathSet &set); -rust::Vec extract_paths(const nix::Store &store, - const nix::StorePaths &set); diff --git a/subprojects/crates/nix-utils/src/cxx/utils.cpp b/subprojects/crates/nix-utils/src/cxx/utils.cpp deleted file mode 100644 index cc73bfb04..000000000 --- a/subprojects/crates/nix-utils/src/cxx/utils.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "nix-utils/include/utils.h" - -#include - -rust::String extract_opt_path(const nix::Store &store, - const std::optional &v) { - // TODO(conni2461): Replace with option - return v ? store.printStorePath(*v) : ""; -} - -rust::Vec extract_path_set(const nix::Store &store, - const nix::StorePathSet &set) { - rust::Vec data; - data.reserve(set.size()); - for (const nix::StorePath &path : set) { - data.emplace_back(store.printStorePath(path)); - } - return data; -} - -rust::Vec extract_paths(const nix::Store &store, - const nix::StorePaths &set) { - rust::Vec data; - data.reserve(set.size()); - for (const nix::StorePath &path : set) { - data.emplace_back(store.printStorePath(path)); - } - return data; -} diff --git a/subprojects/crates/nix-utils/src/drv.rs b/subprojects/crates/nix-utils/src/drv.rs deleted file mode 100644 index 6f1471724..000000000 --- a/subprojects/crates/nix-utils/src/drv.rs +++ /dev/null @@ -1,118 +0,0 @@ -use std::collections::BTreeMap; - -use harmonia_store_core::derived_path::OutputName; -use harmonia_store_core::store_path::{StoreDir, StorePathName}; - -pub use harmonia_store_core::derivation::Derivation; - -use crate::StorePath; - -pub(crate) fn parse_drv( - store_dir: &StoreDir, - drv_path: &StorePath, - input: &str, -) -> Result { - let drv_name_str = drv_path.name().to_string(); - let name: StorePathName = drv_name_str - .strip_suffix(".drv") - .ok_or_else(|| anyhow::anyhow!("derivation path must end in .drv: {drv_name_str}"))? - .parse() - .map_err(|e| anyhow::anyhow!("invalid derivation name: {e}"))?; - - harmonia_store_aterm::parse_derivation_aterm(store_dir, input, name) - .map_err(|e| anyhow::anyhow!("ATerm parse error: {e}").into()) -} - -#[tracing::instrument(skip(store), fields(%drv), err)] -pub async fn query_drv( - store: &crate::LocalStore, - drv: &StorePath, -) -> Result, crate::Error> { - use crate::BaseStore as _; - - if !drv.is_derivation() { - return Ok(None); - } - - if !store.is_valid_path(drv).await { - return Ok(None); - } - - let real_path = store.to_real_path(drv).await?; - let input = fs_err::tokio::read_to_string(&real_path).await?; - Ok(Some(parse_drv(store.get_store_dir(), drv, &input)?)) -} - -/// Resolve output paths for all outputs. Returns `None` for outputs whose -/// paths cannot be determined before building (`Deferred`, `CAFloating`, `Impure`). -pub fn output_paths( - drv: &Derivation, - store_dir: &StoreDir, -) -> BTreeMap> { - drv.outputs - .iter() - .map(|(name, output)| { - let path = output.path(store_dir, &drv.name, name).ok().flatten(); - (name.clone(), path) - }) - .collect() -} - -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used)] - - use harmonia_store_core::derivation::DerivationOutput; - use harmonia_store_core::store_path::StoreDir; - - use crate::drv::parse_drv; - - /// Fake but valid 32-char nix base32 hash for test store paths. - const HASH: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - - fn fake_drv_path(name: &str) -> crate::StorePath { - crate::parse_store_path(&format!("{HASH}-{name}.drv")) - } - - /// Minimal CA fixed-output derivation (fetchurl-style). - #[test] - fn ca_fixed() { - let store_dir = StoreDir::default(); - let drv_path = fake_drv_path("test-src"); - let drv = parse_drv( - &store_dir, - &drv_path, - &format!( - r#"Derive([("out","/nix/store/{HASH}-test-src","sha256","deadbeef00000000000000000000000000000000000000000000000000000000")],[],[],"{0}","{0}",[],[("name","test-src")])"#, - "/bin/sh", - ), - ) - .unwrap(); - - let (_, output) = drv.outputs.iter().next().unwrap(); - assert!(matches!(output, DerivationOutput::CAFixed(_))); - } - - /// Minimal input-addressed derivation with two outputs. - #[test] - fn input_addressed() { - let store_dir = StoreDir::default(); - let drv_path = fake_drv_path("hello-1.0"); - let drv = parse_drv( - &store_dir, - &drv_path, - &format!( - r#"Derive([("lib","/nix/store/{HASH}-hello-1.0-lib","",""),("out","/nix/store/{HASH}-hello-1.0","","")],[],[],"{0}","{0}",[],[("name","hello-1.0")])"#, - "x86_64-linux", - ), - ) - .unwrap(); - - assert_eq!(drv.outputs.len(), 2); - assert!( - drv.outputs - .values() - .all(|o| matches!(o, DerivationOutput::InputAddressed(_))) - ); - } -} diff --git a/subprojects/crates/nix-utils/src/lib.rs b/subprojects/crates/nix-utils/src/lib.rs deleted file mode 100644 index 23414d536..000000000 --- a/subprojects/crates/nix-utils/src/lib.rs +++ /dev/null @@ -1,1214 +0,0 @@ -#![deny( - clippy::all, - clippy::pedantic, - clippy::expect_used, - clippy::unwrap_used, - future_incompatible, - missing_debug_implementations, - nonstandard_style, - unreachable_pub, - missing_copy_implementations, - unused_qualifications -)] -#![allow(clippy::missing_errors_doc)] - -mod drv; -mod realise; -mod store_path; - -use std::collections::{BTreeMap, BTreeSet}; - -use hashbrown::HashMap; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("std io error: `{0}`")] - Io(#[from] std::io::Error), - - #[error("tokio join error: `{0}`")] - TokioJoin(#[from] tokio::task::JoinError), - - #[error("utf8 error: `{0}`")] - Utf8(#[from] std::str::Utf8Error), - - #[error("Failed to get tokio stdout stream")] - Stream, - - #[error("Command failed with `{0}`")] - Exit(std::process::ExitStatus), - - #[error("Exception was thrown `{0}`")] - Exception(#[from] cxx::Exception), - - #[error("anyhow error: `{0}`")] - Anyhow(#[from] anyhow::Error), - - #[error("json error: `{0}`")] - Json(#[from] serde_json::Error), -} - -pub use drv::{Derivation, output_paths, query_drv}; -pub use harmonia_store_core::derivation::{ - BasicDerivation, DerivationOutput, DerivationOutputs, DerivationT, -}; -pub use harmonia_store_core::derived_path::{ - DerivedPath, OutputName, OutputSpec, SingleDerivedPath, -}; -pub use harmonia_store_core::realisation::{DrvOutput, Realisation, UnkeyedRealisation}; -pub use harmonia_store_core::signature::Signature; -pub use realise::{BuildOptions, realise_drv, realise_drvs}; -pub use store_path::{ - ParseStorePathError, StoreDir, StoreDirDisplay, StorePath, StorePathHash, StorePathName, - parse_store_path, -}; - -pub fn validate_statuscode(status: std::process::ExitStatus) -> Result<(), Error> { - if status.success() { - Ok(()) - } else { - Err(Error::Exit(status)) - } -} - -pub fn add_root(store: &LocalStore, root_dir: &std::path::Path, store_path: &StorePath) { - let path = root_dir.join(store_path.to_string()); - // force create symlink - if fs_err::exists(&path).unwrap_or_default() { - let _ = fs_err::remove_file(&path); - } - if !fs_err::exists(&path).unwrap_or_default() { - let target = store.get_store_dir().display(store_path).to_string(); - let _ = fs_err::os::unix::fs::symlink(target, path); - } -} - -#[cxx::bridge(namespace = "nix_utils")] -mod ffi { - #![allow(unreachable_pub, unused_qualifications)] - - #[derive(Debug, Clone)] - struct InternalPathInfo { - deriver: String, - nar_hash: String, - registration_time: i64, - nar_size: u64, - refs: Vec, - sigs: Vec, - ca: String, - } - - #[derive(Debug, Clone, Copy)] - struct StoreStats { - nar_info_read: u64, - nar_info_read_averted: u64, - nar_info_missing: u64, - nar_info_write: u64, - path_info_cache_size: u64, - nar_read: u64, - nar_read_bytes: u64, - nar_read_compressed_bytes: u64, - nar_write: u64, - nar_write_averted: u64, - nar_write_bytes: u64, - nar_write_compressed_bytes: u64, - nar_write_compression_time_ms: u64, - } - - #[derive(Debug, Clone, Copy)] - struct S3Stats { - put: u64, - put_bytes: u64, - put_time_ms: u64, - get: u64, - get_bytes: u64, - get_time_ms: u64, - head: u64, - } - - unsafe extern "C++" { - include!("nix-utils/include/nix.h"); - - type StoreWrapper; - - fn init_nix(); - fn init(uri: &str) -> UniquePtr; - - fn get_store_dir() -> String; - fn get_store_dir_for(store: &StoreWrapper) -> String; - fn get_build_dir() -> String; - fn get_state_dir() -> String; - fn get_nix_version() -> String; - fn get_this_system() -> String; - fn get_extra_platforms() -> Vec; - fn get_system_features() -> Vec; - fn get_substituters() -> Vec; - - fn get_use_cgroups() -> bool; - fn set_verbosity(level: i32); - fn is_valid_path(store: &StoreWrapper, path: &str) -> Result; - fn query_path_info(store: &StoreWrapper, path: &str) -> Result; - fn compute_closure_size(store: &StoreWrapper, path: &str) -> Result; - fn clear_path_info_cache(store: &StoreWrapper) -> Result<()>; - #[allow(clippy::fn_params_excessive_bools)] - fn compute_fs_closure( - store: &StoreWrapper, - path: &str, - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - ) -> Result>; - #[allow(clippy::fn_params_excessive_bools)] - fn compute_fs_closures( - store: &StoreWrapper, - paths: &[&str], - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - toposort: bool, - ) -> Result>; - fn upsert_file(store: &StoreWrapper, path: &str, data: &str, mime_type: &str) - -> Result<()>; - fn get_store_stats(store: &StoreWrapper) -> Result; - fn copy_paths( - src_store: &StoreWrapper, - dst_store: &StoreWrapper, - paths: &[&str], - repair: bool, - check_sigs: bool, - substitute: bool, - ) -> Result<()>; - - fn import_paths( - store: &StoreWrapper, - check_sigs: bool, - runtime: usize, - reader: usize, - callback: unsafe extern "C" fn( - data: &mut [u8], - runtime: usize, - reader: usize, - user_data: usize, - ) -> usize, - user_data: usize, - ) -> Result<()>; - fn import_paths_with_fd(store: &StoreWrapper, check_sigs: bool, fd: i32) -> Result<()>; - fn export_paths( - store: &StoreWrapper, - paths: &[&str], - callback: unsafe extern "C" fn(data: &[u8], user_data: usize) -> bool, - user_data: usize, - ) -> Result<()>; - fn nar_from_path( - store: &StoreWrapper, - paths: &str, - callback: unsafe extern "C" fn(data: &[u8], user_data: usize) -> bool, - user_data: usize, - ) -> Result<()>; - - fn list_nar_deep(store: &StoreWrapper, path: &str) -> Result; - - fn ensure_path(store: &StoreWrapper, path: &str) -> Result<()>; - fn to_real_path(store: &StoreWrapper, path: &str) -> Result; - fn write_derivation(store: &StoreWrapper, json: &str) -> Result; - } -} - -pub use ffi::{S3Stats, StoreStats}; - -impl StoreStats { - #[must_use] - pub fn nar_compression_savings(&self) -> f64 { - #[allow(clippy::cast_precision_loss)] - if self.nar_write_bytes > 0 { - 1.0 - (self.nar_write_compressed_bytes as f64 / self.nar_write_bytes as f64) - } else { - 0.0 - } - } - #[must_use] - pub fn nar_compression_speed(&self) -> f64 { - #[allow(clippy::cast_precision_loss)] - if self.nar_write_compression_time_ms > 0 { - self.nar_write_bytes as f64 / self.nar_write_compression_time_ms as f64 * 1000.0 - / (1024.0 * 1024.0) - } else { - 0.0 - } - } -} - -#[inline] -#[must_use] -pub fn is_subpath(base: &std::path::Path, path: &std::path::Path) -> bool { - path.starts_with(base) -} - -#[inline] -pub fn init_nix() { - ffi::init_nix(); -} - -#[inline] -#[must_use] -pub fn get_store_dir() -> StoreDir { - StoreDir::new(ffi::get_store_dir()).unwrap_or_default() -} - -#[inline] -#[must_use] -pub fn get_build_dir() -> String { - ffi::get_build_dir() -} - -#[inline] -#[must_use] -pub fn get_state_dir() -> String { - ffi::get_state_dir() -} - -#[inline] -#[must_use] -pub fn get_nix_version() -> String { - ffi::get_nix_version() -} - -#[inline] -#[must_use] -pub fn get_this_system() -> String { - ffi::get_this_system() -} - -#[inline] -#[must_use] -pub fn get_extra_platforms() -> Vec { - ffi::get_extra_platforms() -} - -#[inline] -#[must_use] -pub fn get_system_features() -> Vec { - ffi::get_system_features() -} - -#[inline] -#[must_use] -pub fn get_substituters() -> Vec { - ffi::get_substituters() -} - -#[inline] -#[must_use] -pub fn get_use_cgroups() -> bool { - ffi::get_use_cgroups() -} - -#[inline] -/// Set the loglevel. -pub fn set_verbosity(level: i32) { - ffi::set_verbosity(level); -} - -pub(crate) async fn asyncify(f: F) -> Result -where - F: FnOnce() -> Result + Send + 'static, - T: Send + 'static, -{ - match tokio::task::spawn_blocking(f).await { - Ok(res) => Ok(res?), - Err(_) => Err(std::io::Error::other("background task failed"))?, - } -} - -#[inline] -pub async fn copy_paths( - src: &BaseStoreImpl, - dst: &BaseStoreImpl, - paths: &[StorePath], - repair: bool, - check_sigs: bool, - substitute: bool, -) -> Result<(), Error> { - let paths = paths - .iter() - .map(|p| src.print_store_path(p)) - .collect::>(); - - let src = src.wrapper.clone(); - let dst = dst.wrapper.clone(); - - asyncify(move || { - let slice = paths.iter().map(String::as_str).collect::>(); - ffi::copy_paths( - src.as_raw(), - dst.as_raw(), - &slice, - repair, - check_sigs, - substitute, - ) - }) - .await -} - -#[derive(Debug)] -pub struct PathInfo { - pub deriver: Option, - pub nar_hash: String, - pub registration_time: i64, - pub nar_size: u64, - pub refs: Vec, - pub sigs: Vec, - pub ca: Option, -} - -impl From for PathInfo { - fn from(val: ffi::InternalPathInfo) -> Self { - Self { - deriver: if val.deriver.is_empty() { - None - } else { - Some(parse_store_path(&val.deriver)) - }, - nar_hash: val.nar_hash, - registration_time: val.registration_time, - nar_size: val.nar_size, - refs: val.refs.iter().map(|v| parse_store_path(v)).collect(), - sigs: val.sigs, - ca: if val.ca.is_empty() { - None - } else { - Some(val.ca) - }, - } - } -} - -pub trait BaseStore { - #[must_use] - /// Check whether a path is valid. - fn is_valid_path(&self, path: &StorePath) -> impl Future; - - fn query_path_info(&self, path: &StorePath) -> impl Future>; - fn query_path_infos( - &self, - paths: &[&StorePath], - ) -> impl Future>; - fn compute_closure_size(&self, path: &StorePath) -> impl Future; - - fn clear_path_info_cache(&self); - - #[allow(clippy::fn_params_excessive_bools)] - fn compute_fs_closure( - &self, - path: &str, - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - ) -> Result, cxx::Exception>; - - #[allow(clippy::fn_params_excessive_bools)] - fn compute_fs_closures( - &self, - paths: &[&StorePath], - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - toposort: bool, - ) -> impl Future, Error>>; - - fn query_requisites( - &self, - drvs: &[&StorePath], - include_outputs: bool, - ) -> impl Future, Error>>; - - fn get_store_stats(&self) -> Result; - - /// Import paths from nar - fn import_paths( - &self, - stream: S, - check_sigs: bool, - ) -> impl Future> - where - S: tokio_stream::Stream> - + Send - + Unpin - + 'static; - - /// Import paths from nar - fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> - where - Fd: std::os::fd::AsFd + std::os::fd::AsRawFd; - - /// Export a store path in NAR format. The data is passed in chunks to callback - fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool; - - /// Export a store path in NAR format. The data is passed in chunks to callback - fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool; - - fn list_nar_deep(&self, path: &StorePath) -> impl Future>; - - fn ensure_path(&self, path: &StorePath) -> impl Future>; - - #[must_use] - fn store_dir(&self) -> &StoreDir; - - #[must_use] - fn print_store_path(&self, path: &StorePath) -> String { - self.store_dir().display(path).to_string() - } -} - -struct FFIStore(std::cell::UnsafeCell>); -unsafe impl Send for FFIStore {} -unsafe impl Sync for FFIStore {} - -impl FFIStore { - fn as_raw(&self) -> &ffi::StoreWrapper { - unsafe { &*self.0.get() } - } - - #[allow(unused, clippy::mut_from_ref)] - fn as_pin_mut(&self) -> std::pin::Pin<&mut ffi::StoreWrapper> { - let ptr = unsafe { &mut *self.0.get() }; - ptr.pin_mut() - } -} - -#[derive(Clone)] -#[allow(missing_debug_implementations)] -pub struct BaseStoreImpl { - wrapper: std::sync::Arc, - store_dir: StoreDir, -} - -impl BaseStoreImpl { - fn new(store: cxx::UniquePtr) -> Self { - let store_dir = StoreDir::new(ffi::get_store_dir_for(&store)).unwrap_or_default(); - Self { - wrapper: std::sync::Arc::new(FFIStore(std::cell::UnsafeCell::new(store))), - store_dir, - } - } -} - -fn import_paths_trampoline( - data: &mut [u8], - runtime: usize, - reader: usize, - userdata: usize, -) -> usize -where - F: FnMut( - &tokio::runtime::Runtime, - &mut Box>, - &mut [u8], - ) -> usize, - S: futures::stream::Stream>, - E: Into, -{ - let runtime = - unsafe { &*(runtime as *mut std::ffi::c_void).cast::>() }; - let reader = unsafe { - &mut *(reader as *mut std::ffi::c_void) - .cast::>>() - }; - let closure = unsafe { &mut *(userdata as *mut std::ffi::c_void).cast::() }; - closure(runtime, reader, data) -} - -fn export_paths_trampoline(data: &[u8], userdata: usize) -> bool -where - F: FnMut(&[u8]) -> bool, -{ - let closure = unsafe { &mut *(userdata as *mut std::ffi::c_void).cast::() }; - closure(data) -} - -impl BaseStore for BaseStoreImpl { - #[inline] - async fn is_valid_path(&self, path: &StorePath) -> bool { - let store = self.wrapper.clone(); - let path = self.print_store_path(path); - asyncify(move || ffi::is_valid_path(store.as_raw(), &path)) - .await - .unwrap_or(false) - } - - #[inline] - async fn query_path_info(&self, path: &StorePath) -> Option { - let store = self.wrapper.clone(); - let path = self.print_store_path(path); - asyncify(move || { - Ok(ffi::query_path_info(store.as_raw(), &path) - .ok() - .map(Into::into)) - }) - .await - .ok() - .flatten() - } - - #[inline] - async fn query_path_infos(&self, paths: &[&StorePath]) -> HashMap { - let paths = paths.iter().map(|v| (*v).to_owned()).collect::>(); - - asyncify({ - let self_ = self.clone(); - move || { - let mut res = HashMap::with_capacity(paths.len()); - for p in paths { - let full_path = self_.print_store_path(&p); - if let Some(info) = ffi::query_path_info(self_.wrapper.as_raw(), &full_path) - .ok() - .map(Into::into) - { - res.insert(p, info); - } - } - Ok(res) - } - }) - .await - .unwrap_or_default() - } - - #[inline] - async fn compute_closure_size(&self, path: &StorePath) -> u64 { - let store = self.wrapper.clone(); - let path = self.print_store_path(path); - asyncify(move || ffi::compute_closure_size(store.as_raw(), &path)) - .await - .unwrap_or_default() - } - - #[inline] - fn clear_path_info_cache(&self) { - let _ = ffi::clear_path_info_cache(self.wrapper.as_raw()); - } - - #[inline] - #[tracing::instrument(skip(self), err)] - fn compute_fs_closure( - &self, - path: &str, - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - ) -> Result, cxx::Exception> { - ffi::compute_fs_closure( - self.wrapper.as_raw(), - path, - flip_direction, - include_outputs, - include_derivers, - ) - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn compute_fs_closures( - &self, - paths: &[&StorePath], - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - toposort: bool, - ) -> Result, Error> { - let store = self.wrapper.clone(); - let paths = paths - .iter() - .map(|v| self.print_store_path(v)) - .collect::>(); - - asyncify(move || { - let slice = paths.iter().map(String::as_str).collect::>(); - Ok(ffi::compute_fs_closures( - store.as_raw(), - &slice, - flip_direction, - include_outputs, - include_derivers, - toposort, - )? - .into_iter() - .map(|v| parse_store_path(&v)) - .collect()) - }) - .await - } - - async fn query_requisites( - &self, - drvs: &[&StorePath], - include_outputs: bool, - ) -> Result, Error> { - let mut out = self - .compute_fs_closures(drvs, false, include_outputs, false, true) - .await?; - out.reverse(); - Ok(out) - } - - fn get_store_stats(&self) -> Result { - ffi::get_store_stats(self.wrapper.as_raw()) - } - - #[inline] - #[tracing::instrument(skip(self, stream), err)] - async fn import_paths(&self, stream: S, check_sigs: bool) -> Result<(), Error> - where - S: tokio_stream::Stream> - + Send - + Unpin - + 'static, - { - use tokio::io::AsyncReadExt as _; - - let callback = |runtime: &tokio::runtime::Runtime, - reader: &mut Box>, - data: &mut [u8]| { - runtime.block_on(async { reader.read(data).await.unwrap_or(0) }) - }; - - let reader = Box::new(tokio_util::io::StreamReader::new(stream)); - let store = self.clone(); - tokio::task::spawn_blocking(move || { - store.import_paths_with_cb(callback, reader, check_sigs) - }) - .await??; - Ok(()) - } - - #[inline] - #[tracing::instrument(skip(self, fd), err)] - fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> - where - Fd: std::os::fd::AsFd + std::os::fd::AsRawFd, - { - ffi::import_paths_with_fd(self.wrapper.as_raw(), check_sigs, fd.as_raw_fd()) - } - - #[inline] - #[tracing::instrument(skip(self, paths, callback), err)] - fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool, - { - let paths = paths - .iter() - .map(|v| self.print_store_path(v)) - .collect::>(); - let slice = paths.iter().map(String::as_str).collect::>(); - ffi::export_paths( - self.wrapper.as_raw(), - &slice, - export_paths_trampoline::, - std::ptr::addr_of!(callback).cast::() as usize, - ) - } - - #[inline] - #[tracing::instrument(skip(self, path, callback), err)] - fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool, - { - let path = self.print_store_path(path); - ffi::nar_from_path( - self.wrapper.as_raw(), - &path, - export_paths_trampoline::, - std::ptr::addr_of!(callback).cast::() as usize, - ) - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn list_nar_deep(&self, path: &StorePath) -> Result { - let store = self.wrapper.clone(); - let path = self.print_store_path(path); - asyncify(move || ffi::list_nar_deep(store.as_raw(), &path)).await - } - - #[inline] - async fn ensure_path(&self, path: &StorePath) -> Result<(), Error> { - let store = self.wrapper.clone(); - let path = self.print_store_path(path); - asyncify(move || { - ffi::ensure_path(store.as_raw(), &path)?; - Ok(()) - }) - .await - } - - #[inline] - fn store_dir(&self) -> &StoreDir { - &self.store_dir - } -} - -impl BaseStoreImpl { - #[inline] - #[tracing::instrument(skip(self, callback, reader), err)] - fn import_paths_with_cb( - &self, - callback: F, - reader: Box>, - check_sigs: bool, - ) -> Result<(), Error> - where - F: FnMut( - &tokio::runtime::Runtime, - &mut Box>, - &mut [u8], - ) -> usize, - S: futures::stream::Stream>, - E: Into, - { - let runtime = Box::new(tokio::runtime::Runtime::new()?); - ffi::import_paths( - self.wrapper.as_raw(), - check_sigs, - std::ptr::addr_of!(runtime).cast::() as usize, - std::ptr::addr_of!(reader).cast::() as usize, - import_paths_trampoline::, - std::ptr::addr_of!(callback).cast::() as usize, - )?; - drop(reader); - drop(runtime); - Ok(()) - } -} - -#[derive(Clone)] -#[allow(missing_debug_implementations)] -pub struct LocalStore { - base: BaseStoreImpl, -} - -impl LocalStore { - #[inline] - /// Initialise a new store - #[must_use] - pub fn init() -> Self { - let base = BaseStoreImpl::new(ffi::init("")); - Self { base } - } - - #[must_use] - pub const fn as_base_store(&self) -> &BaseStoreImpl { - &self.base - } - - #[tracing::instrument(skip(self, outputs))] - pub async fn query_missing_outputs( - &self, - outputs: BTreeMap>, - ) -> BTreeMap> { - use futures::stream::StreamExt as _; - - tokio_stream::iter(outputs) - .map(|(name, path)| async move { - match path { - Some(p) if self.is_valid_path(&p).await => None, - other => Some((name, other)), - } - }) - .buffered(50) - .filter_map(|o| async { o }) - .collect() - .await - } - - #[must_use] - pub fn get_store_dir(&self) -> &StoreDir { - &self.base.store_dir - } - - pub fn unsafe_set_store_dir(&mut self, store_dir: StoreDir) { - self.base.store_dir = store_dir; - } - - /// Write a [`BasicDerivation`] to the store. - /// - /// Returns the store path of the written `.drv` file. - pub async fn write_derivation(&self, drv: &BasicDerivation) -> Result { - let full_drv: DerivationT> = drv - .clone() - .map_inputs(|inputs| inputs.into_iter().map(SingleDerivedPath::Opaque).collect()); - let json = serde_json::to_string(&full_drv) - .map_err(|e| anyhow::anyhow!("failed to serialize derivation: {e}"))?; - let store = self.base.wrapper.clone(); - asyncify(move || { - let path = ffi::write_derivation(store.as_raw(), &json)?; - Ok(parse_store_path(&path)) - }) - .await - } - - /// Resolve a store path to its physical filesystem location. - /// - /// For stores where the physical store directory differs from the - /// logical `storeDir` (e.g. `local?root=X&store=Y`), this returns the - /// actual on-disk path rather than the logical store path. - pub async fn to_real_path(&self, path: &StorePath) -> Result { - let printed = self.base.store_dir.display(path).to_string(); - let store = self.base.wrapper.clone(); - asyncify(move || ffi::to_real_path(store.as_raw(), &printed)).await - } -} - -impl BaseStore for LocalStore { - #[inline] - async fn is_valid_path(&self, path: &StorePath) -> bool { - self.base.is_valid_path(path).await - } - - #[inline] - async fn query_path_info(&self, path: &StorePath) -> Option { - self.base.query_path_info(path).await - } - - #[inline] - async fn query_path_infos(&self, paths: &[&StorePath]) -> HashMap { - self.base.query_path_infos(paths).await - } - - #[inline] - async fn compute_closure_size(&self, path: &StorePath) -> u64 { - self.base.compute_closure_size(path).await - } - - #[inline] - fn clear_path_info_cache(&self) { - self.base.clear_path_info_cache(); - } - - #[inline] - #[tracing::instrument(skip(self), err)] - fn compute_fs_closure( - &self, - path: &str, - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - ) -> Result, cxx::Exception> { - self.base - .compute_fs_closure(path, flip_direction, include_outputs, include_derivers) - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn compute_fs_closures( - &self, - paths: &[&StorePath], - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - toposort: bool, - ) -> Result, Error> { - self.base - .compute_fs_closures( - paths, - flip_direction, - include_outputs, - include_derivers, - toposort, - ) - .await - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn query_requisites( - &self, - drvs: &[&StorePath], - include_outputs: bool, - ) -> Result, Error> { - self.base.query_requisites(drvs, include_outputs).await - } - - #[inline] - fn get_store_stats(&self) -> Result { - self.base.get_store_stats() - } - - #[inline] - #[tracing::instrument(skip(self, stream), err)] - async fn import_paths(&self, stream: S, check_sigs: bool) -> Result<(), Error> - where - S: tokio_stream::Stream> - + Send - + Unpin - + 'static, - { - self.base.import_paths::(stream, check_sigs).await - } - - #[inline] - #[tracing::instrument(skip(self, fd), err)] - fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> - where - Fd: std::os::fd::AsFd + std::os::fd::AsRawFd, - { - self.base.import_paths_with_fd(fd, check_sigs) - } - - #[inline] - #[tracing::instrument(skip(self, paths, callback), err)] - fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool, - { - self.base.export_paths(paths, callback) - } - - #[inline] - #[tracing::instrument(skip(self, path, callback), err)] - fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool, - { - self.base.nar_from_path(path, callback) - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn list_nar_deep(&self, path: &StorePath) -> Result { - self.base.list_nar_deep(path).await - } - - #[inline] - async fn ensure_path(&self, path: &StorePath) -> Result<(), Error> { - self.base.ensure_path(path).await - } - - #[inline] - fn store_dir(&self) -> &StoreDir { - self.base.store_dir() - } -} - -#[derive(Clone)] -#[allow(missing_debug_implementations)] -pub struct RemoteStore { - base: BaseStoreImpl, - - pub uri: String, - pub base_uri: String, -} - -impl RemoteStore { - #[inline] - /// Initialise a new store with uri - #[must_use] - pub fn init(uri: &str) -> Self { - let base_uri = url::Url::parse(uri) - .ok() - .and_then(|v| v.host_str().map(ToOwned::to_owned)) - .unwrap_or_default(); - - Self { - base: BaseStoreImpl::new(ffi::init(uri)), - uri: uri.into(), - base_uri, - } - } - - #[must_use] - pub const fn as_base_store(&self) -> &BaseStoreImpl { - &self.base - } - - #[inline] - pub async fn upsert_file( - &self, - path: String, - local_path: std::path::PathBuf, - mime_type: &'static str, - ) -> Result<(), Error> { - let store = self.base.wrapper.clone(); - asyncify(move || { - if let Ok(data) = fs_err::read_to_string(local_path) { - ffi::upsert_file(store.as_raw(), &path, &data, mime_type)?; - } - Ok(()) - }) - .await - } - - #[tracing::instrument(skip(self, paths))] - pub async fn query_missing_paths(&self, paths: Vec) -> Vec { - use futures::stream::StreamExt as _; - - tokio_stream::iter(paths) - .map(|p| async move { - if self.is_valid_path(&p).await { - None - } else { - Some(p) - } - }) - .buffered(50) - .filter_map(|p| async { p }) - .collect() - .await - } - - #[tracing::instrument(skip(self, outputs))] - pub async fn query_missing_remote_outputs( - &self, - outputs: BTreeMap>, - ) -> BTreeMap> { - use futures::stream::StreamExt as _; - - tokio_stream::iter(outputs) - .map(|(name, path)| async move { - match path { - Some(p) if self.is_valid_path(&p).await => None, - other => Some((name, other)), - } - }) - .buffered(50) - .filter_map(|o| async { o }) - .collect() - .await - } -} - -impl BaseStore for RemoteStore { - #[inline] - async fn is_valid_path(&self, path: &StorePath) -> bool { - self.base.is_valid_path(path).await - } - - #[inline] - async fn query_path_info(&self, path: &StorePath) -> Option { - self.base.query_path_info(path).await - } - - #[inline] - async fn query_path_infos(&self, paths: &[&StorePath]) -> HashMap { - self.base.query_path_infos(paths).await - } - - #[inline] - async fn compute_closure_size(&self, path: &StorePath) -> u64 { - self.base.compute_closure_size(path).await - } - - #[inline] - fn clear_path_info_cache(&self) { - self.base.clear_path_info_cache(); - } - - #[inline] - #[tracing::instrument(skip(self), err)] - fn compute_fs_closure( - &self, - path: &str, - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - ) -> Result, cxx::Exception> { - self.base - .compute_fs_closure(path, flip_direction, include_outputs, include_derivers) - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn compute_fs_closures( - &self, - paths: &[&StorePath], - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - toposort: bool, - ) -> Result, Error> { - self.base - .compute_fs_closures( - paths, - flip_direction, - include_outputs, - include_derivers, - toposort, - ) - .await - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn query_requisites( - &self, - drvs: &[&StorePath], - include_outputs: bool, - ) -> Result, Error> { - self.base.query_requisites(drvs, include_outputs).await - } - - #[inline] - fn get_store_stats(&self) -> Result { - self.base.get_store_stats() - } - - #[inline] - #[tracing::instrument(skip(self, stream), err)] - async fn import_paths(&self, stream: S, check_sigs: bool) -> Result<(), Error> - where - S: tokio_stream::Stream> - + Send - + Unpin - + 'static, - { - self.base.import_paths::(stream, check_sigs).await - } - - #[inline] - #[tracing::instrument(skip(self, fd), err)] - fn import_paths_with_fd(&self, fd: Fd, check_sigs: bool) -> Result<(), cxx::Exception> - where - Fd: std::os::fd::AsFd + std::os::fd::AsRawFd, - { - self.base.import_paths_with_fd(fd, check_sigs) - } - - #[inline] - #[tracing::instrument(skip(self, paths, callback), err)] - fn export_paths(&self, paths: &[StorePath], callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool, - { - self.base.export_paths(paths, callback) - } - - #[inline] - #[tracing::instrument(skip(self, path, callback), err)] - fn nar_from_path(&self, path: &StorePath, callback: F) -> Result<(), cxx::Exception> - where - F: FnMut(&[u8]) -> bool, - { - self.base.nar_from_path(path, callback) - } - - #[inline] - #[tracing::instrument(skip(self), err)] - async fn list_nar_deep(&self, path: &StorePath) -> Result { - self.base.list_nar_deep(path).await - } - - #[inline] - async fn ensure_path(&self, path: &StorePath) -> Result<(), Error> { - self.base.ensure_path(path).await - } - - #[inline] - fn store_dir(&self) -> &StoreDir { - self.base.store_dir() - } -} diff --git a/subprojects/crates/nix-utils/src/nix.cpp b/subprojects/crates/nix-utils/src/nix.cpp deleted file mode 100644 index 6c6e0f7c4..000000000 --- a/subprojects/crates/nix-utils/src/nix.cpp +++ /dev/null @@ -1,348 +0,0 @@ -#include "nix-utils/include/nix.h" -#include "nix-utils/include/utils.h" - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "nix/store/export-import.hh" -#include -#include -#include -#include -#include - -#include - -static std::atomic initializedNix = false; -static std::mutex nixInitMtx; - -namespace nix_utils { -void init_nix() { - if (!initializedNix) { - // We need this mutex here. if we have multiple threads that want to do - // init_nix at the same time. - // We need to ensure that initNix is finished on all threads before setting - // initializedNix. - // We also need to ensure that initNix not runs multiple times at the same - // time - std::lock_guard lock(nixInitMtx); - if (!initializedNix) { - nix::initNix(); - initializedNix = true; - } - } -} - -StoreWrapper::StoreWrapper(nix::ref _store) : _store(_store) {} - -std::unique_ptr init(rust::Str uri) { - init_nix(); - if (uri.empty()) { - nix::ref _store = nix::openStore(); - return std::make_unique(_store); - } else { - nix::ref _store = nix::openStore(AS_STRING(uri)); - return std::make_unique(_store); - } -} - -rust::String get_store_dir() { - init_nix(); - return nix::openStore()->storeDir; -} -rust::String get_store_dir_for(const StoreWrapper &wrapper) { - return wrapper._store->storeDir; -} -rust::String get_build_dir() { - auto &localSettings = nix::settings.getLocalSettings(); - auto buildDir = localSettings.buildDir.get(); - return buildDir.has_value() - ? buildDir->string() - : (nix::settings.nixStateDir / "builds").string(); -} -rust::String get_state_dir() { return nix::settings.nixStateDir.string(); } -rust::String get_nix_version() { return nix::nixVersion; } -rust::String get_this_system() { return nix::settings.thisSystem.get(); } -rust::Vec get_extra_platforms() { - auto set = nix::settings.extraPlatforms.get(); - rust::Vec data; - data.reserve(set.size()); - for (const auto &val : set) { - data.emplace_back(val); - } - return data; -} -rust::Vec get_system_features() { - auto set = nix::settings.systemFeatures.get(); - rust::Vec data; - data.reserve(set.size()); - for (const auto &val : set) { - data.emplace_back(val); - } - return data; -} -rust::Vec get_substituters() { - auto refs = nix::settings.getWorkerSettings().substituters.get(); - rust::Vec data; - data.reserve(refs.size()); - for (const auto &val : refs) { - data.emplace_back(val.render()); - } - return data; -} - -bool get_use_cgroups() { -#ifdef __linux__ - return nix::settings.getLocalSettings().useCgroups; -#endif - return false; -} -void set_verbosity(int32_t level) { nix::verbosity = (nix::Verbosity)level; } - -bool is_valid_path(const StoreWrapper &wrapper, rust::Str path) { - auto store = wrapper._store; - return store->isValidPath(store->parseStorePath(AS_VIEW(path))); -} - -InternalPathInfo query_path_info(const StoreWrapper &wrapper, rust::Str path) { - auto store = wrapper._store; - auto info = store->queryPathInfo(store->parseStorePath(AS_VIEW(path))); - - std::string narhash = info->narHash.to_string(nix::HashFormat::Nix32, true); - - rust::Vec refs = extract_path_set(*store, info->references); - - rust::Vec sigs; - sigs.reserve(info->sigs.size()); - for (const auto &sig : info->sigs) { - sigs.push_back(sig.to_string()); - } - - // TODO(conni2461): Replace "" with option - return InternalPathInfo{ - extract_opt_path(*store, info->deriver), - narhash, - info->registrationTime, - info->narSize, - refs, - sigs, - info->ca ? nix::renderContentAddress(*info->ca) : "", - }; -} - -uint64_t compute_closure_size(const StoreWrapper &wrapper, rust::Str path) { - auto store = wrapper._store; - nix::StorePathSet closure; - store->computeFSClosure(store->parseStorePath(AS_VIEW(path)), closure, false, - false); - - uint64_t totalNarSize = 0; - for (auto &p : closure) { - totalNarSize += store->queryPathInfo(p)->narSize; - } - return totalNarSize; -} - -void clear_path_info_cache(const StoreWrapper &wrapper) { - auto store = wrapper._store; - store->clearPathInfoCache(); -} - -rust::Vec compute_fs_closure(const StoreWrapper &wrapper, - rust::Str path, bool flip_direction, - bool include_outputs, - bool include_derivers) { - auto store = wrapper._store; - nix::StorePathSet path_set; - store->computeFSClosure(store->parseStorePath(AS_VIEW(path)), path_set, - flip_direction, include_outputs, include_derivers); - return extract_path_set(*store, path_set); -} - -rust::Vec compute_fs_closures(const StoreWrapper &wrapper, - rust::Slice paths, - bool flip_direction, - bool include_outputs, - bool include_derivers, - bool toposort) { - auto store = wrapper._store; - nix::StorePathSet path_set; - for (auto &path : paths) { - store->computeFSClosure(store->parseStorePath(AS_VIEW(path)), path_set, - flip_direction, include_outputs, include_derivers); - } - if (toposort) { - auto sorted = store->topoSortPaths(path_set); - return extract_paths(*store, sorted); - } else { - return extract_path_set(*store, path_set); - } -} - -void upsert_file(const StoreWrapper &wrapper, rust::Str path, rust::Str data, - rust::Str mime_type) { - auto store = wrapper._store.dynamic_pointer_cast(); - if (!store) { - throw nix::Error("Not a binary chache store"); - } - store->upsertFile(AS_STRING(path), AS_STRING(data), AS_STRING(mime_type)); -} - -StoreStats get_store_stats(const StoreWrapper &wrapper) { - auto store = wrapper._store; - auto &stats = store->getStats(); - return StoreStats{ - stats.narInfoRead.load(), - stats.narInfoReadAverted.load(), - stats.narInfoMissing.load(), - stats.narInfoWrite.load(), - stats.pathInfoCacheSize.load(), - stats.narRead.load(), - stats.narReadBytes.load(), - stats.narReadCompressedBytes.load(), - stats.narWrite.load(), - stats.narWriteAverted.load(), - stats.narWriteBytes.load(), - stats.narWriteCompressedBytes.load(), - stats.narWriteCompressionTimeMs.load(), - }; -} - -void copy_paths(const StoreWrapper &src_store, const StoreWrapper &dst_store, - rust::Slice paths, bool repair, - bool check_sigs, bool substitute) { - nix::StorePathSet path_set; - for (auto &path : paths) { - path_set.insert(src_store._store->parseStorePath(AS_VIEW(path))); - } - nix::copyPaths(*src_store._store, *dst_store._store, path_set, - repair ? nix::Repair : nix::NoRepair, - check_sigs ? nix::CheckSigs : nix::NoCheckSigs, - substitute ? nix::Substitute : nix::NoSubstitute); -} - -void import_paths( - const StoreWrapper &wrapper, bool check_sigs, size_t runtime, size_t reader, - rust::Fn, size_t, size_t, size_t)> callback, - size_t user_data) { - nix::LambdaSource source([=](char *out, size_t out_len) { - auto data = rust::Slice((uint8_t *)out, out_len); - size_t ret = (*callback)(data, runtime, reader, user_data); - if (!ret) { - throw nix::EndOfFile("End of stream reached"); - } - return ret; - }); - - auto store = wrapper._store; - auto paths = nix::importPaths(*store, source, - check_sigs ? nix::CheckSigs : nix::NoCheckSigs); -} - -void import_paths_with_fd(const StoreWrapper &wrapper, bool check_sigs, - int32_t fd) { - nix::FdSource source(fd); - - auto store = wrapper._store; - nix::importPaths(*store, source, - check_sigs ? nix::CheckSigs : nix::NoCheckSigs); -} - -class StopExport : public std::exception { -public: - const char *what() { return "Stop exporting nar"; } -}; - -void export_paths(const StoreWrapper &wrapper, - rust::Slice paths, - rust::Fn, size_t)> callback, - size_t user_data) { - nix::LambdaSink sink([=](std::string_view v) { - auto data = rust::Slice((const uint8_t *)v.data(), v.size()); - bool ret = (*callback)(data, user_data); - if (!ret) { - throw StopExport(); - } - }); - - auto store = wrapper._store; - nix::StorePathSet path_set; - for (auto &path : paths) { - path_set.insert(store->followLinksToStorePath(AS_VIEW(path))); - } - try { - nix::exportPaths(*store, path_set, sink); - } catch (StopExport &e) { - // Intentionally do nothing. We're only using the exception as a - // short-circuiting mechanism. - } -} - -void nar_from_path(const StoreWrapper &wrapper, rust::Str path, - rust::Fn, size_t)> callback, - size_t user_data) { - nix::LambdaSink sink([=](std::string_view v) { - auto data = rust::Slice((const uint8_t *)v.data(), v.size()); - bool ret = (*callback)(data, user_data); - if (!ret) { - throw StopExport(); - } - }); - - auto store = wrapper._store; - try { - store->narFromPath(store->followLinksToStorePath(AS_VIEW(path)), sink); - } catch (StopExport &e) { - // Intentionally do nothing. We're only using the exception as a - // short-circuiting mechanism. - } -} - -rust::String list_nar_deep(const StoreWrapper &wrapper, rust::Str path) { - auto store = wrapper._store; - auto [store_path, rest] = store->toStorePath(AS_VIEW(path)); - - nlohmann::json j = { - {"version", 1}, - {"root", nix::listNarDeep(*store->getFSAccessor(), - nix::CanonPath{store_path.to_string()} / - nix::CanonPath{rest})}, - }; - - return j.dump(); -} - -void ensure_path(const StoreWrapper &wrapper, rust::Str path) { - auto store = wrapper._store; - store->ensurePath(store->followLinksToStorePath(AS_VIEW(path))); -} - -rust::String to_real_path(const StoreWrapper &wrapper, rust::Str path) { - auto store = wrapper._store; - auto *lfs = dynamic_cast(&*store); - if (!lfs) { - throw nix::Error( - "toRealPath: store '%s' is not a local filesystem store", - store->config.getHumanReadableURI()); - } - auto storePath = store->parseStorePath(AS_VIEW(path)); - auto s = lfs->toRealPath(storePath).string(); - return rust::String(s.data(), s.size()); -} - -rust::String write_derivation(const StoreWrapper &wrapper, rust::Str json) { - auto store = wrapper._store; - auto drv = nix::Derivation::parseJsonAndValidate( - *store, nlohmann::json::parse(AS_VIEW(json))); - auto path = store->writeDerivation(drv, nix::NoRepair); - auto s = path.to_string(); - return rust::String(s.data(), s.size()); -} -} // namespace nix_utils diff --git a/subprojects/crates/nix-utils/src/realise.rs b/subprojects/crates/nix-utils/src/realise.rs deleted file mode 100644 index 879e01eb0..000000000 --- a/subprojects/crates/nix-utils/src/realise.rs +++ /dev/null @@ -1,151 +0,0 @@ -use std::sync::Arc; - -use tokio::io::{AsyncBufReadExt as _, BufReader}; -use tokio_stream::wrappers::LinesStream; - -use crate::BaseStore as _; -use crate::{DerivedPath, OutputSpec, SingleDerivedPath, StorePath}; - -#[derive(Debug, Clone, Copy)] -pub struct BuildOptions { - max_log_size: u64, - max_silent_time: i32, - build_timeout: i32, - check: bool, -} - -impl BuildOptions { - #[must_use] - pub fn new(max_log_size: Option) -> Self { - Self { - max_log_size: max_log_size.unwrap_or(64u64 << 20), - max_silent_time: 0, - build_timeout: 0, - check: false, - } - } - - #[must_use] - pub const fn complete(max_log_size: u64, max_silent_time: i32, build_timeout: i32) -> Self { - Self { - max_log_size, - max_silent_time, - build_timeout, - check: false, - } - } - - pub const fn set_max_silent_time(&mut self, max_silent_time: i32) { - self.max_silent_time = max_silent_time; - } - - pub const fn set_build_timeout(&mut self, build_timeout: i32) { - self.build_timeout = build_timeout; - } - - #[must_use] - pub const fn get_max_log_size(&self) -> u64 { - self.max_log_size - } - - #[must_use] - pub const fn get_max_silent_time(&self) -> i32 { - self.max_silent_time - } - - #[must_use] - pub const fn get_build_timeout(&self) -> i32 { - self.build_timeout - } - - #[must_use] - pub const fn enable_check_build(mut self) -> Self { - self.check = true; - self - } -} - -#[allow(clippy::type_complexity)] -#[tracing::instrument(skip(store, opts, drvs), err)] -pub async fn realise_drvs( - store: &crate::LocalStore, - drvs: &[&StorePath], - opts: &BuildOptions, - kill_on_drop: bool, -) -> Result< - ( - tokio::process::Child, - LinesStream>, - LinesStream>, - ), - crate::Error, -> { - let mut child = tokio::process::Command::new("nix") - .args([ - "--extra-experimental-features", - "nix-command", - "build", - "--json", - "--no-pretty", - "--print-build-logs", - "--log-format", - "raw-with-logs", - "--no-link", - "--max-silent-time", - &opts.max_silent_time.to_string(), - "--timeout", - &opts.build_timeout.to_string(), - "--option", - "max-build-log-size", - &opts.max_log_size.to_string(), - "--option", - "fallback", - "true", - "--option", - "substitute", - "false", - "--option", - "builders", - "", - ]) - .args(if opts.check { vec!["--check"] } else { vec![] }) - .args(drvs.iter().map(|v| { - store - .store_dir() - .display(&DerivedPath::Built { - drv_path: Arc::new(SingleDerivedPath::Opaque((*v).clone())), - outputs: OutputSpec::All, - }) - .to_string() - })) - .kill_on_drop(kill_on_drop) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn()?; - - let stdout = child.stdout.take().ok_or(crate::Error::Stream)?; - let stderr = child.stderr.take().ok_or(crate::Error::Stream)?; - - let stdout = LinesStream::new(BufReader::new(stdout).lines()); - let stderr = LinesStream::new(BufReader::new(stderr).lines()); - - Ok((child, stdout, stderr)) -} - -#[allow(clippy::type_complexity)] -#[tracing::instrument(skip(store, opts), fields(%drv), err)] -pub async fn realise_drv( - store: &crate::LocalStore, - drv: &StorePath, - opts: &BuildOptions, - kill_on_drop: bool, -) -> Result< - ( - tokio::process::Child, - LinesStream>, - LinesStream>, - ), - crate::Error, -> { - realise_drvs(store, &[drv], opts, kill_on_drop).await -} diff --git a/subprojects/crates/nix-utils/src/store_path.rs b/subprojects/crates/nix-utils/src/store_path.rs deleted file mode 100644 index bc6b2a4c9..000000000 --- a/subprojects/crates/nix-utils/src/store_path.rs +++ /dev/null @@ -1,55 +0,0 @@ -#[allow(unreachable_pub)] -pub use harmonia_store_core::store_path::{ - ParseStorePathError, StoreDir, StoreDirDisplay, StorePath, StorePathHash, StorePathName, -}; - -/// Parse a store path from a string that may or may not have the store dir prefix. -/// Handles paths inside store outputs (e.g. `/nix/store/hash-name/subdir/file`). -#[must_use] -pub fn parse_store_path(s: &str) -> StorePath { - let after_store = s.find("/store/").map_or(s, |i| &s[i + 7..]); - let base = after_store.split('/').next().unwrap_or(after_store); - base.parse() - .unwrap_or_else(|e| panic!("invalid store path '{s}': {e}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_base_name() { - let sp = parse_store_path("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package-name"); - assert_eq!( - sp.to_string(), - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package-name" - ); - } - - #[test] - fn test_parse_with_store_prefix() { - let sp = parse_store_path("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package-name"); - assert_eq!( - sp.to_string(), - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package-name" - ); - } - - #[test] - fn test_parse_with_subpath() { - let sp = - parse_store_path("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package-name/bin/hello"); - assert_eq!( - sp.to_string(), - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package-name" - ); - } - - #[test] - fn test_is_drv() { - let drv = parse_store_path("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package.drv"); - let regular = parse_store_path("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-package"); - assert!(drv.is_derivation()); - assert!(!regular.is_derivation()); - } -} diff --git a/subprojects/crates/proto/Cargo.toml b/subprojects/crates/proto/Cargo.toml new file mode 100644 index 000000000..5b16a62fc --- /dev/null +++ b/subprojects/crates/proto/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "hydra-proto" +version.workspace = true +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +bytes.workspace = true +harmonia-store-content-address.workspace = true +harmonia-store-derivation.workspace = true +harmonia-store-nar-info.workspace = true +harmonia-store-path.workspace = true +harmonia-store-path-info.workspace = true +harmonia-utils-hash.workspace = true +harmonia-utils-signature.workspace = true +prost.workspace = true +serde_json.workspace = true +tonic.workspace = true +tonic-prost.workspace = true + +db = { workspace = true, optional = true } +nix-support.workspace = true +store-path-utils.workspace = true + +[features] +client = [ ] +db = [ "dep:db" ] +server = [ ] + +[build-dependencies] +fs-err = { workspace = true } +sha2.workspace = true +thiserror.workspace = true +tonic-prost-build.workspace = true diff --git a/subprojects/crates/proto/build.rs b/subprojects/crates/proto/build.rs new file mode 100644 index 000000000..bf1a74512 --- /dev/null +++ b/subprojects/crates/proto/build.rs @@ -0,0 +1,75 @@ +use sha2::Digest; +use std::{env, path::PathBuf}; + +#[derive(Debug, thiserror::Error)] +enum BuildError { + #[error(transparent)] + Var(#[from] env::VarError), + #[error(transparent)] + Io(#[from] std::io::Error), +} + +fn main() -> Result<(), BuildError> { + let out_dir = PathBuf::from(env::var("OUT_DIR")?); + + let workspace_version = env::var("CARGO_PKG_VERSION")?; + + println!("cargo:rerun-if-changed=../../proto/v1/streaming.proto"); + println!("cargo:rerun-if-changed=../../proto/v1/store.proto"); + println!("cargo:rerun-if-changed=../../proto/v1/store/derivation.proto"); + println!("cargo:rerun-if-changed=../../proto/v1/nix-support.proto"); + + let mut hasher = sha2::Sha256::new(); + hasher.update(fs_err::read_to_string("../../proto/v1/streaming.proto")?.as_bytes()); + hasher.update(fs_err::read_to_string("../../proto/v1/nix-support.proto")?.as_bytes()); + let proto_hash = format!("{:x}", hasher.finalize()); + let version = format!("{}-{}", workspace_version, &proto_hash[..8]); + + fs_err::write( + out_dir.join("proto_version.rs"), + format!( + r#"// Generated during build - do not edit +pub const PROTO_API_VERSION: &str = "{version}"; +"# + ), + )?; + + // First pass: generate nix.store.v1 types (except StorePath which is manual) + tonic_prost_build::configure() + .extern_path( + ".nix.store.v1.StorePath", + "crate::store_path::ProtoStorePath", + ) + .build_client(false) + .build_server(false) + .compile_protos( + &[ + "../../proto/v1/store.proto", + "../../proto/v1/store/derivation.proto", + ], + &["../../proto"], + )?; + + // Second pass: generate runner.v1 (references nix.store.v1 via extern_path) + tonic_prost_build::configure() + .extern_path( + ".nix.store.v1.StorePath", + "crate::store_path::ProtoStorePath", + ) + .extern_path(".nix.store.v1", "crate::nix::store::v1") + .extern_path( + ".nix.store.derivation.v1", + "crate::nix::store::derivation::v1", + ) + .build_client(cfg!(feature = "client")) + .build_server(cfg!(feature = "server")) + .file_descriptor_set_path(out_dir.join("streaming_descriptor.bin")) + .compile_protos( + &[ + "../../proto/v1/nix-support.proto", + "../../proto/v1/streaming.proto", + ], + &["../../proto"], + )?; + Ok(()) +} diff --git a/subprojects/crates/proto/src/lib.rs b/subprojects/crates/proto/src/lib.rs new file mode 100644 index 000000000..62d4b4c54 --- /dev/null +++ b/subprojects/crates/proto/src/lib.rs @@ -0,0 +1,612 @@ +// We need to allow pedantic here because of generated code +#![allow(clippy::pedantic, unused_qualifications)] + +use harmonia_utils_hash::HashView; + +pub mod store_path; + +pub use store_path::ProtoStorePath; + +pub mod nix { + pub mod store { + pub mod v1 { + tonic::include_proto!("nix.store.v1"); + } + pub mod derivation { + pub mod v1 { + tonic::include_proto!("nix.store.derivation.v1"); + } + } + } +} + +pub use nix::store::v1::{ + NarInfo, RelativeStorePath, StorePaths, UnkeyedNarInfo, UnkeyedValidPathInfo, ValidPathInfo, +}; + +tonic::include_proto!("runner.v1"); + +include!(concat!(env!("OUT_DIR"), "/proto_version.rs")); + +pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("streaming_descriptor"); + +impl From<&store_path_utils::RelativeStorePath> for RelativeStorePath { + fn from(r: &store_path_utils::RelativeStorePath) -> Self { + Self { + store_path: Some(ProtoStorePath::from(&r.base_path)), + sub_path: r.relative_path.to_string(), + } + } +} + +impl TryFrom for store_path_utils::RelativeStorePath { + type Error = &'static str; + + fn try_from(r: RelativeStorePath) -> Result { + let store_path = r.store_path.ok_or("missing store_path")?; + Ok(Self { + base_path: store_path.0, + relative_path: r.sub_path.into(), + }) + } +} + +// -- Conversions between proto types and harmonia types -- + +/// Error type for converting proto types to harmonia types. +#[derive(Debug, Clone)] +pub struct NarInfoConvertError(pub &'static str); + +impl std::fmt::Display for NarInfoConvertError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } +} + +impl std::error::Error for NarInfoConvertError {} + +// -- Hash Algorithm -- + +impl From for nix::store::v1::hash::Algorithm { + fn from(algo: harmonia_utils_hash::Algorithm) -> Self { + use harmonia_utils_hash::Algorithm; + match algo { + Algorithm::SHA256 => Self::Sha256, + Algorithm::SHA512 => Self::Sha512, + Algorithm::SHA1 => Self::Sha1, + Algorithm::MD5 => Self::Md5, + Algorithm::BLAKE3 => Self::Blake3, + } + } +} + +impl From for harmonia_utils_hash::Algorithm { + fn from(algo: nix::store::v1::hash::Algorithm) -> Self { + match algo { + nix::store::v1::hash::Algorithm::Sha256 => Self::SHA256, + nix::store::v1::hash::Algorithm::Sha512 => Self::SHA512, + nix::store::v1::hash::Algorithm::Sha1 => Self::SHA1, + nix::store::v1::hash::Algorithm::Md5 => Self::MD5, + nix::store::v1::hash::Algorithm::Blake3 => Self::BLAKE3, + } + } +} + +// -- Hash -- + +impl From<&harmonia_utils_hash::Hash> for nix::store::v1::Hash { + fn from(h: &harmonia_utils_hash::Hash) -> Self { + Self { + algorithm: nix::store::v1::hash::Algorithm::from(h.algorithm()) as i32, + digest: h.as_ref().to_vec(), + } + } +} + +impl TryFrom for harmonia_utils_hash::Hash { + type Error = &'static str; + + fn try_from(h: nix::store::v1::Hash) -> Result { + let algo: harmonia_utils_hash::Algorithm = + nix::store::v1::hash::Algorithm::try_from(h.algorithm) + .map_err(|_| "unknown hash algorithm")? + .into(); + harmonia_utils_hash::Hash::from_slice(algo, &h.digest) + .map_err(|_| "invalid hash digest length") + } +} + +// -- Signature -- + +impl From<&harmonia_utils_signature::Signature> for nix::store::v1::Signature { + fn from(sig: &harmonia_utils_signature::Signature) -> Self { + Self { + key_name: sig.key_name.clone(), + sig: sig.sig.to_string(), + } + } +} + +impl TryFrom for harmonia_utils_signature::Signature { + type Error = &'static str; + + fn try_from(sig: nix::store::v1::Signature) -> Result { + Ok(Self { + key_name: sig.key_name.clone(), + sig: sig.sig.parse().map_err(|_| "invalid signature")?, + }) + } +} + +// -- ContentAddress -- + +impl From<&harmonia_store_content_address::ContentAddress> for nix::store::v1::ContentAddress { + fn from(ca: &harmonia_store_content_address::ContentAddress) -> Self { + use harmonia_store_content_address::ContentAddress as CA; + match ca { + CA::Text(h) => Self { + method: nix::store::v1::content_address::Method::Text as i32, + hash: Some(nix::store::v1::Hash::from( + &harmonia_utils_hash::Hash::from(*h), + )), + }, + CA::Flat(h) => Self { + method: nix::store::v1::content_address::Method::Flat as i32, + hash: Some(nix::store::v1::Hash::from(h)), + }, + CA::NixArchive(h) => Self { + method: nix::store::v1::content_address::Method::NixArchive as i32, + hash: Some(nix::store::v1::Hash::from(h)), + }, + } + } +} + +impl TryFrom for harmonia_store_content_address::ContentAddress { + type Error = &'static str; + + fn try_from(ca: nix::store::v1::ContentAddress) -> Result { + use harmonia_store_content_address::ContentAddress as CA; + let hash: harmonia_utils_hash::Hash = ca.hash.ok_or("missing CA hash")?.try_into()?; + match nix::store::v1::content_address::Method::try_from(ca.method) { + Ok(nix::store::v1::content_address::Method::Text) => { + let sha256 = harmonia_utils_hash::Sha256::try_from(hash) + .map_err(|_| "Text CA requires SHA256")?; + Ok(CA::Text(sha256)) + } + Ok(nix::store::v1::content_address::Method::Flat) => Ok(CA::Flat(hash)), + Ok(nix::store::v1::content_address::Method::NixArchive) => Ok(CA::NixArchive(hash)), + Err(_) => Err("unknown CA method"), + } + } +} + +// -- ContentAddressMethodAlgorithm -- + +impl From<&harmonia_store_content_address::ContentAddressMethodAlgorithm> + for nix::store::v1::ContentAddressMethodAlgorithm +{ + fn from(ca: &harmonia_store_content_address::ContentAddressMethodAlgorithm) -> Self { + use harmonia_store_content_address::ContentAddressMethodAlgorithm as CAMA; + match ca { + CAMA::Text => Self { + method: nix::store::v1::content_address::Method::Text as i32, + algorithm: nix::store::v1::hash::Algorithm::Sha256 as i32, + }, + CAMA::Flat(algo) => Self { + method: nix::store::v1::content_address::Method::Flat as i32, + algorithm: nix::store::v1::hash::Algorithm::from(*algo) as i32, + }, + CAMA::NixArchive(algo) => Self { + method: nix::store::v1::content_address::Method::NixArchive as i32, + algorithm: nix::store::v1::hash::Algorithm::from(*algo) as i32, + }, + } + } +} + +impl TryFrom + for harmonia_store_content_address::ContentAddressMethodAlgorithm +{ + type Error = &'static str; + + fn try_from(ca: nix::store::v1::ContentAddressMethodAlgorithm) -> Result { + use harmonia_store_content_address::ContentAddressMethodAlgorithm as CAMA; + let method = nix::store::v1::content_address::Method::try_from(ca.method) + .map_err(|_| "unknown content address method")?; + let algo: harmonia_utils_hash::Algorithm = + nix::store::v1::hash::Algorithm::try_from(ca.algorithm) + .map_err(|_| "unknown hash algorithm")? + .into(); + match method { + nix::store::v1::content_address::Method::Text => Ok(CAMA::Text), + nix::store::v1::content_address::Method::Flat => Ok(CAMA::Flat(algo)), + nix::store::v1::content_address::Method::NixArchive => Ok(CAMA::NixArchive(algo)), + } + } +} + +// -- DerivationOutput -- + +impl From<&harmonia_store_derivation::derivation::DerivationOutput> + for nix::store::derivation::v1::Output +{ + fn from(o: &harmonia_store_derivation::derivation::DerivationOutput) -> Self { + use harmonia_store_derivation::derivation::DerivationOutput as DO; + use nix::store::derivation::v1::output::Output; + Self { + output: Some(match o { + DO::InputAddressed(p) => Output::InputAddressed(ProtoStorePath::from(p)), + DO::CAFixed(ca) => Output::CaFixed(nix::store::v1::ContentAddress::from(ca)), + DO::Deferred => Output::Deferred(nix::store::derivation::v1::Unit {}), + DO::CAFloating(cama) => { + Output::CaFloating(nix::store::v1::ContentAddressMethodAlgorithm::from(cama)) + } + DO::Impure(cama) => { + Output::Impure(nix::store::v1::ContentAddressMethodAlgorithm::from(cama)) + } + }), + } + } +} + +impl TryFrom + for harmonia_store_derivation::derivation::DerivationOutput +{ + type Error = &'static str; + + fn try_from(o: nix::store::derivation::v1::Output) -> Result { + use harmonia_store_derivation::derivation::DerivationOutput as DO; + use nix::store::derivation::v1::output::Output; + match o.output.ok_or("missing DerivationOutput variant")? { + Output::InputAddressed(p) => Ok(DO::InputAddressed(p.0)), + Output::CaFixed(ca) => Ok(DO::CAFixed(ca.try_into()?)), + Output::Deferred(_) => Ok(DO::Deferred), + Output::CaFloating(cama) => Ok(DO::CAFloating(cama.try_into()?)), + Output::Impure(cama) => Ok(DO::Impure(cama.try_into()?)), + } + } +} + +// -- BasicDerivation -- + +impl From<&harmonia_store_derivation::derivation::BasicDerivation> + for nix::store::derivation::v1::Basic +{ + fn from(d: &harmonia_store_derivation::derivation::BasicDerivation) -> Self { + Self { + name: d.name.to_string(), + outputs: d + .outputs + .iter() + .map(|(name, output)| { + ( + name.to_string(), + nix::store::derivation::v1::Output::from(output), + ) + }) + .collect(), + inputs: d.inputs.iter().map(ProtoStorePath::from).collect(), + platform: d.platform.to_vec(), + builder: d.builder.to_vec(), + args: d.args.iter().map(|a| a.to_vec()).collect(), + env: d + .env + .iter() + .map(|(k, v)| (std::str::from_utf8(k).unwrap_or("").to_owned(), v.to_vec())) + .collect(), + structured_attrs: d + .structured_attrs + .as_ref() + .map(|sa| serde_json::to_string(&sa.attrs).unwrap_or_default()), + } + } +} + +impl TryFrom + for harmonia_store_derivation::derivation::BasicDerivation +{ + type Error = String; + + fn try_from(d: nix::store::derivation::v1::Basic) -> Result { + use bytes::Bytes as ByteString; + use harmonia_store_derivation::derivation::{DerivationOutput, StructuredAttrs}; + use harmonia_store_derivation::derived_path::OutputName; + + let name = d + .name + .parse() + .map_err(|e| format!("invalid derivation name: {e}"))?; + + let outputs = d + .outputs + .into_iter() + .map(|(k, v)| { + let name: OutputName = + k.parse().map_err(|e| format!("invalid output name: {e}"))?; + let output: DerivationOutput = v.try_into().map_err(|e: &str| e.to_owned())?; + Ok((name, output)) + }) + .collect::>()?; + + let inputs = d.inputs.into_iter().map(|p| p.0).collect(); + + let structured_attrs = d + .structured_attrs + .map(|s| { + let attrs: serde_json::Map = serde_json::from_str(&s) + .map_err(|e| format!("invalid structured_attrs JSON: {e}"))?; + Ok::<_, String>(StructuredAttrs { attrs }) + }) + .transpose()?; + + Ok(harmonia_store_derivation::derivation::DerivationT { + name, + outputs, + inputs, + platform: ByteString::from(d.platform), + builder: ByteString::from(d.builder), + args: d.args.into_iter().map(ByteString::from).collect(), + env: d + .env + .into_iter() + .map(|(k, v)| (ByteString::from(k), ByteString::from(v))) + .collect(), + structured_attrs, + }) + } +} + +// -- UnkeyedValidPathInfo -- + +impl From<&harmonia_store_path_info::UnkeyedValidPathInfo> for UnkeyedValidPathInfo { + fn from(v: &harmonia_store_path_info::UnkeyedValidPathInfo) -> Self { + let nar_hash: harmonia_utils_hash::Hash = v.nar_hash.into(); + Self { + deriver: v.deriver.as_ref().map(ProtoStorePath::from), + nar_hash: Some(nix::store::v1::Hash::from(&nar_hash)), + references: v.references.iter().map(ProtoStorePath::from).collect(), + registration_time: v.registration_time.map(|t| t.get()), + nar_size: v.nar_size, + ultimate: v.ultimate, + signatures: v + .signatures + .iter() + .map(nix::store::v1::Signature::from) + .collect(), + ca: v.ca.as_ref().map(nix::store::v1::ContentAddress::from), + store_dir: v.store_dir.to_string(), + } + } +} + +impl TryFrom for harmonia_store_path_info::UnkeyedValidPathInfo { + type Error = NarInfoConvertError; + + fn try_from(v: UnkeyedValidPathInfo) -> Result { + let raw_hash: harmonia_utils_hash::Hash = v + .nar_hash + .ok_or(NarInfoConvertError("missing nar_hash"))? + .try_into() + .map_err(|_| NarInfoConvertError("invalid nar_hash"))?; + let nar_hash = harmonia_store_path_info::NarHash::try_from(raw_hash) + .map_err(|_| NarInfoConvertError("nar_hash is not sha256"))?; + + Ok(Self { + deriver: v.deriver.map(|p| p.0), + nar_hash, + references: v.references.into_iter().map(|p| p.0).collect(), + registration_time: std::num::NonZero::new(v.registration_time.unwrap_or(0)), + nar_size: v.nar_size, + ultimate: v.ultimate, + signatures: v + .signatures + .into_iter() + .filter_map(|s| harmonia_utils_signature::Signature::try_from(s).ok()) + .collect(), + ca: v + .ca + .map(harmonia_store_content_address::ContentAddress::try_from) + .transpose() + .map_err(|_| NarInfoConvertError("invalid ca"))?, + store_dir: harmonia_store_path::StoreDir::new(&v.store_dir).unwrap_or_default(), + }) + } +} + +// -- ValidPathInfo (keyed) -- + +impl + From<( + &harmonia_store_path::StorePath, + &harmonia_store_path_info::UnkeyedValidPathInfo, + )> for ValidPathInfo +{ + fn from( + (path, info): ( + &harmonia_store_path::StorePath, + &harmonia_store_path_info::UnkeyedValidPathInfo, + ), + ) -> Self { + Self { + path: Some(ProtoStorePath::from(path)), + info: Some(UnkeyedValidPathInfo::from(info)), + } + } +} + +// -- UnkeyedNarInfo -- + +impl From<&harmonia_store_nar_info::UnkeyedNarInfo> for UnkeyedNarInfo { + fn from(n: &harmonia_store_nar_info::UnkeyedNarInfo) -> Self { + Self { + info: Some(UnkeyedValidPathInfo::from(&n.info)), + url: n.url.clone().unwrap_or_default(), + compression: n.compression.clone().unwrap_or_default(), + download_hash: n.download_hash.as_ref().map(nix::store::v1::Hash::from), + download_size: n.download_size, + } + } +} + +impl TryFrom for harmonia_store_nar_info::UnkeyedNarInfo { + type Error = NarInfoConvertError; + + fn try_from(n: UnkeyedNarInfo) -> Result { + let info = n + .info + .ok_or(NarInfoConvertError("missing info"))? + .try_into()?; + Ok(Self { + info, + url: if n.url.is_empty() { None } else { Some(n.url) }, + compression: if n.compression.is_empty() { + None + } else { + Some(n.compression) + }, + download_hash: n + .download_hash + .map(harmonia_utils_hash::Hash::try_from) + .transpose() + .map_err(|_| NarInfoConvertError("invalid download_hash"))?, + download_size: n.download_size, + }) + } +} + +// -- NarInfo -- + +impl From<&harmonia_store_nar_info::NarInfo> for NarInfo { + fn from(n: &harmonia_store_nar_info::NarInfo) -> Self { + Self { + path: Some(ProtoStorePath::from(n.path.clone())), + info: Some(UnkeyedNarInfo::from(&n.info)), + } + } +} + +impl TryFrom for harmonia_store_nar_info::NarInfo { + type Error = NarInfoConvertError; + + fn try_from(n: NarInfo) -> Result { + let path = n.path.ok_or(NarInfoConvertError("missing path"))?.0; + let info = n + .info + .ok_or(NarInfoConvertError("missing info"))? + .try_into()?; + Ok(Self { path, info }) + } +} + +// -- BuildProduct -- + +impl From for BuildProduct { + fn from(p: nix_support::BuildProduct) -> Self { + Self { + path: Some((&p.path).into()), + default_path: p.default_path, + r#type: p.r#type, + subtype: p.subtype, + name: p.name, + is_regular: p.is_regular, + sha256hash: p.sha256hash.map(|h| { + harmonia_utils_hash::fmt::Bare::>::from(h) + .to_string() + }), + file_size: p.file_size, + } + } +} + +impl TryFrom for nix_support::BuildProduct { + type Error = &'static str; + + fn try_from(p: BuildProduct) -> Result { + let path: store_path_utils::RelativeStorePath = + p.path.ok_or("BuildProduct missing path")?.try_into()?; + Ok(Self { + path, + default_path: p.default_path, + r#type: p.r#type, + subtype: p.subtype, + name: p.name, + is_regular: p.is_regular, + sha256hash: p.sha256hash.as_deref().and_then(|s| { + s.parse::, + >>() + .ok() + .map(Into::into) + }), + file_size: p.file_size, + }) + } +} + +// -- BuildMetric -- + +impl From for BuildMetric { + fn from(m: nix_support::BuildMetric) -> Self { + Self { + unit: m.unit, + value: m.value, + } + } +} + +impl From for nix_support::BuildMetric { + fn from(m: BuildMetric) -> Self { + Self { + unit: m.unit, + value: m.value, + } + } +} + +// -- NixSupport -- + +impl From for NixSupport { + fn from(ns: nix_support::NixSupport) -> Self { + Self { + failed: ns.failed, + hydra_release_name: ns.hydra_release_name, + metrics: ns.metrics.into_iter().map(|(k, v)| (k, v.into())).collect(), + products: ns.products.into_iter().map(Into::into).collect(), + } + } +} + +impl From for nix_support::NixSupport { + fn from(ns: NixSupport) -> Self { + Self { + failed: ns.failed, + hydra_release_name: ns.hydra_release_name, + metrics: ns.metrics.into_iter().map(|(k, v)| (k, v.into())).collect(), + products: ns + .products + .into_iter() + .map(TryInto::try_into) + .collect::>() + .unwrap_or_default(), + } + } +} + +#[cfg(test)] +mod tests; + +#[cfg(feature = "db")] +impl From for db::models::StepStatus { + fn from(item: StepStatus) -> Self { + match item { + StepStatus::Preparing => Self::Preparing, + StepStatus::Connecting => Self::Connecting, + StepStatus::SeningInputs => Self::SendingInputs, + StepStatus::Building => Self::Building, + StepStatus::WaitingForLocalSlot => Self::WaitingForLocalSlot, + StepStatus::ReceivingOutputs => Self::ReceivingOutputs, + StepStatus::PostProcessing => Self::PostProcessing, + } + } +} diff --git a/subprojects/crates/shared/src/proto.rs b/subprojects/crates/proto/src/store_path.rs similarity index 95% rename from subprojects/crates/shared/src/proto.rs rename to subprojects/crates/proto/src/store_path.rs index 810dd68e6..82842d869 100644 --- a/subprojects/crates/shared/src/proto.rs +++ b/subprojects/crates/proto/src/store_path.rs @@ -1,7 +1,7 @@ //! Newtype wrappers for using harmonia store types with prost/tonic. //! //! These exist to work around the orphan rule: we cannot implement -//! `prost::Message` for `harmonia_store_core::store_path::StorePath` +//! `prost::Message` for `harmonia_store_path::StorePath` //! directly because both the trait and the type are foreign. Instead, //! we define thin newtypes here (inside the hydra workspace) and map //! them via `extern_path` in the prost-build configuration. @@ -10,7 +10,7 @@ use prost::DecodeError; use prost::bytes::{Buf, BufMut}; use prost::encoding::{self, DecodeContext, WireType}; -use nix_utils::StorePath; +use harmonia_store_path::StorePath; /// A [`StorePath`] that implements [`prost::Message`]. /// @@ -40,6 +40,12 @@ impl From for ProtoStorePath { } } +impl From<&StorePath> for ProtoStorePath { + fn from(p: &StorePath) -> Self { + Self(p.clone()) + } +} + impl From for StorePath { fn from(p: ProtoStorePath) -> StorePath { p.0 diff --git a/subprojects/crates/proto/src/tests.rs b/subprojects/crates/proto/src/tests.rs new file mode 100644 index 000000000..7567012b1 --- /dev/null +++ b/subprojects/crates/proto/src/tests.rs @@ -0,0 +1,323 @@ +use super::*; +use std::collections::{BTreeMap, BTreeSet}; + +use harmonia_store_content_address::ContentAddress; +use harmonia_store_path::{StoreDir, StorePath}; +use harmonia_store_path_info::NarHash; +use harmonia_utils_hash::{Algorithm, Hash, Sha256}; +use harmonia_utils_signature::Signature; + +use super::{BuildMetric, BuildProduct, NixSupport}; + +fn test_store_dir() -> StoreDir { + StoreDir::default() +} + +fn test_sig() -> Signature { + "cache.nixos.org-1:0CpHca+06TwFp9VkMyz5OaphT3E8mnS+1SWymYlvFaghKSYPCMQ66TS1XPAr1+y9rfQZPLaHrBjjnIRktE/nAA==".parse().unwrap() +} + +fn test_path() -> StorePath { + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello".parse().unwrap() +} + +fn test_unkeyed_valid_path_info() -> harmonia_store_path_info::UnkeyedValidPathInfo { + harmonia_store_path_info::UnkeyedValidPathInfo { + deriver: Some("cccccccccccccccccccccccccccccccc-drv.drv".parse().unwrap()), + nar_hash: NarHash::from_slice(&[0xab; 32]).unwrap(), + references: BTreeSet::from(["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-dep".parse().unwrap()]), + registration_time: std::num::NonZero::new(1700000000), + nar_size: 42000, + ultimate: true, + signatures: BTreeSet::from([test_sig()]), + ca: Some(ContentAddress::NixArchive( + Hash::from_slice(Algorithm::SHA256, &[0x55; 32]).unwrap(), + )), + store_dir: test_store_dir(), + } +} + +fn roundtrip(original: &A) -> A +where + for<'a> &'a A: Into, + B: TryInto, + >::Error: std::fmt::Debug, +{ + let proto: B = original.into(); + proto.try_into().unwrap() +} + +// -- Hash round-trips -- + +#[test] +fn hash_roundtrip_sha256() { + let h = Hash::from_slice(Algorithm::SHA256, &[0xab; 32]).unwrap(); + assert_eq!(roundtrip::(&h), h); +} + +#[test] +fn hash_roundtrip_sha512() { + let h = Hash::from_slice(Algorithm::SHA512, &[0xcd; 64]).unwrap(); + assert_eq!(roundtrip::(&h), h); +} + +#[test] +fn signature_roundtrip() { + let sig: Signature = test_sig(); + assert_eq!(roundtrip::(&sig), sig); +} + +#[test] +fn content_address_roundtrip_text() { + let ca = ContentAddress::Text(Sha256::from_slice(&[0x11; 32]).unwrap()); + assert_eq!( + roundtrip::(&ca), + ca + ); +} + +#[test] +fn content_address_roundtrip_flat() { + let ca = ContentAddress::Flat(Hash::from_slice(Algorithm::SHA256, &[0x22; 32]).unwrap()); + assert_eq!( + roundtrip::(&ca), + ca + ); +} + +#[test] +fn content_address_roundtrip_nar() { + let ca = ContentAddress::NixArchive(Hash::from_slice(Algorithm::SHA256, &[0x33; 32]).unwrap()); + assert_eq!( + roundtrip::(&ca), + ca + ); +} + +// -- RelativeStorePath round-trips -- + +#[test] +fn relative_store_path_roundtrip() { + let rsp = store_path_utils::RelativeStorePath { + base_path: test_path(), + relative_path: "share/doc/manual".into(), + }; + assert_eq!( + roundtrip::(&rsp), + rsp + ); +} + +#[test] +fn relative_store_path_roundtrip_bare() { + let rsp = store_path_utils::RelativeStorePath { + base_path: test_path(), + relative_path: "".into(), + }; + assert_eq!( + roundtrip::(&rsp), + rsp + ); +} + +// -- UnkeyedValidPathInfo round-trips -- + +#[test] +fn unkeyed_valid_path_info_roundtrip() { + let v = test_unkeyed_valid_path_info(); + assert_eq!( + roundtrip::(&v), + v + ); +} + +// -- UnkeyedNarInfo round-trips -- + +#[test] +fn unkeyed_nar_info_roundtrip() { + let v = harmonia_store_nar_info::UnkeyedNarInfo { + info: test_unkeyed_valid_path_info(), + url: Some("nar/foo.nar.zst".to_owned()), + compression: Some("zstd".to_owned()), + download_hash: Some(Hash::from_slice(Algorithm::SHA256, &[0xee; 32]).unwrap()), + download_size: Some(500), + }; + assert_eq!( + roundtrip::(&v), + v + ); +} + +// -- NarInfo round-trips -- + +#[test] +fn narinfo_roundtrip_minimal() { + let ni = harmonia_store_nar_info::NarInfo { + path: test_path(), + info: harmonia_store_nar_info::UnkeyedNarInfo { + info: harmonia_store_path_info::UnkeyedValidPathInfo { + deriver: None, + nar_hash: NarHash::from_slice(&[0xab; 32]).unwrap(), + references: BTreeSet::from(["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-dep" + .parse() + .unwrap()]), + registration_time: None, + nar_size: 12345, + ultimate: false, + signatures: BTreeSet::new(), + ca: None, + store_dir: test_store_dir(), + }, + url: Some("nar/abc.nar".to_owned()), + compression: Some("zstd".to_owned()), + download_hash: None, + download_size: None, + }, + }; + assert_eq!( + roundtrip::(&ni), + ni + ); +} + +#[test] +fn narinfo_roundtrip_with_all_fields() { + let ni = harmonia_store_nar_info::NarInfo { + path: test_path(), + info: harmonia_store_nar_info::UnkeyedNarInfo { + info: test_unkeyed_valid_path_info(), + url: Some("nar/xyz.nar.zst".to_owned()), + compression: Some("zstd".to_owned()), + download_hash: Some(Hash::from_slice(Algorithm::SHA256, &[0xee; 32]).unwrap()), + download_size: Some(9999), + }, + }; + assert_eq!( + roundtrip::(&ni), + ni + ); +} + +#[test] +fn narinfo_missing_path_fails() { + let proto = NarInfo { + path: None, + info: Some(nix::store::v1::UnkeyedNarInfo { + info: None, + url: String::new(), + compression: String::new(), + download_hash: None, + download_size: None, + }), + }; + let result: Result = proto.try_into(); + assert!(result.is_err()); +} + +#[test] +fn narinfo_missing_info_fails() { + let proto = NarInfo { + path: Some(ProtoStorePath::from(test_path())), + info: None, + }; + let result: Result = proto.try_into(); + assert!(result.is_err()); +} + +// -- NixSupport round-trips -- + +fn owned_roundtrip(original: A) -> A +where + A: Clone + Into, + B: TryInto, + >::Error: std::fmt::Debug, +{ + let proto: B = original.clone().into(); + proto.try_into().unwrap() +} + +#[test] +fn build_product_roundtrip() { + let bp = nix_support::BuildProduct { + path: store_path_utils::RelativeStorePath { + base_path: test_path(), + relative_path: "bin/hello".into(), + }, + default_path: "bin/hello".into(), + r#type: "doc".into(), + subtype: "readme".into(), + name: "README.md".into(), + is_regular: true, + sha256hash: Some(Sha256::from_slice(&[0xab; 32]).unwrap()), + file_size: Some(1024), + }; + assert_eq!( + owned_roundtrip::(bp.clone()), + bp + ); +} + +#[test] +fn build_product_roundtrip_no_hash() { + let bp = nix_support::BuildProduct { + path: store_path_utils::RelativeStorePath { + base_path: test_path(), + relative_path: "".into(), + }, + default_path: String::new(), + r#type: "nix-build".into(), + subtype: "out-path".into(), + name: "output".into(), + is_regular: false, + sha256hash: None, + file_size: None, + }; + assert_eq!( + owned_roundtrip::(bp.clone()), + bp + ); +} + +#[test] +fn build_metric_roundtrip() { + let bm = nix_support::BuildMetric { + unit: Some("seconds".into()), + value: 42.5, + }; + assert_eq!( + owned_roundtrip::(bm.clone()), + bm + ); +} + +#[test] +fn nix_support_roundtrip() { + let ns = nix_support::NixSupport { + failed: false, + hydra_release_name: Some("hello-1.0".into()), + metrics: BTreeMap::from([( + "build_time".into(), + nix_support::BuildMetric { + unit: Some("seconds".into()), + value: 42.5, + }, + )]), + products: vec![nix_support::BuildProduct { + path: store_path_utils::RelativeStorePath { + base_path: test_path(), + relative_path: "bin/hello".into(), + }, + default_path: "bin/hello".into(), + r#type: "doc".into(), + subtype: "readme".into(), + name: "README.md".into(), + is_regular: true, + sha256hash: Some(Sha256::from_slice(&[0xab; 32]).unwrap()), + file_size: Some(1024), + }], + }; + assert_eq!( + owned_roundtrip::(ns.clone()), + ns + ); +} diff --git a/subprojects/crates/shared/src/lib.rs b/subprojects/crates/shared/src/lib.rs deleted file mode 100644 index f727e6f48..000000000 --- a/subprojects/crates/shared/src/lib.rs +++ /dev/null @@ -1,456 +0,0 @@ -#![forbid(unsafe_code)] -#![deny( - clippy::all, - clippy::pedantic, - clippy::expect_used, - clippy::unwrap_used, - future_incompatible, - missing_debug_implementations, - nonstandard_style, - unreachable_pub, - missing_copy_implementations, - unused_qualifications -)] -#![allow(clippy::missing_errors_doc)] - -pub mod proto; - -use std::{os::unix::fs::MetadataExt as _, sync::LazyLock}; - -use std::collections::BTreeMap; - -use sha2::{Digest as _, Sha256}; -use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, BufReader}; - -use nix_utils::{BaseStore as _, StorePath}; - -#[allow(clippy::expect_used)] -static VALIDATE_METRICS_NAME: LazyLock = - LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9._-]+").expect("Failed to compile regex")); -#[allow(clippy::expect_used)] -static VALIDATE_METRICS_UNIT: LazyLock = - LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9._%-]+").expect("Failed to compile regex")); -#[allow(clippy::expect_used)] -static VALIDATE_RELEASE_NAME: LazyLock = - LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9.@:_-]+").expect("Failed to compile regex")); -#[allow(clippy::expect_used)] -static VALIDATE_PRODUCT_NAME: LazyLock = - LazyLock::new(|| regex::Regex::new("[a-zA-Z0-9.@:_ -]*").expect("Failed to compile regex")); -#[allow(clippy::expect_used)] -static BUILD_PRODUCT_PARSER: LazyLock = LazyLock::new(|| { - regex::Regex::new( - r#"([a-zA-Z0-9_-]+)\s+([a-zA-Z0-9_-]+)\s+(\"[^\"]+\"|[^\"\s<>]+)(\s+([^\"\s<>]+))?"#, - ) - .expect("Failed to compile regex") -}); - -#[derive(Debug)] -pub struct BuildProduct { - pub path: String, - pub default_path: String, - - pub r#type: String, - pub subtype: String, - pub name: String, - - pub is_regular: bool, - - pub sha256hash: Option, - pub file_size: Option, -} - -#[derive(Debug)] -pub struct BuildMetric { - pub path: String, - pub name: String, - pub unit: Option, - pub value: f64, -} - -#[derive(Debug)] -pub struct NixSupport { - pub failed: bool, - pub hydra_release_name: Option, - pub metrics: Vec, - pub products: Vec, -} - -#[derive(Debug, Clone)] -struct FileMetadata { - is_regular: bool, - size: u64, -} - -trait FsOperations { - fn is_inside_store(&self, path: &std::path::Path) -> bool; - fn get_metadata( - &self, - path: impl AsRef + std::fmt::Debug, - ) -> impl Future>; - fn get_file_hash( - &self, - path: impl Into + std::fmt::Debug, - ) -> impl Future>; -} - -#[derive(Debug, Clone)] -struct FilesystemOperations { - store_dir: nix_utils::StoreDir, -} - -impl FilesystemOperations { - fn new() -> Self { - Self { - store_dir: nix_utils::get_store_dir(), - } - } -} - -impl FsOperations for FilesystemOperations { - fn is_inside_store(&self, path: &std::path::Path) -> bool { - nix_utils::is_subpath(self.store_dir.to_path(), path) - } - - #[tracing::instrument(skip(self), err)] - async fn get_metadata( - &self, - path: impl AsRef + std::fmt::Debug, - ) -> Result { - let m = fs_err::tokio::metadata(path).await?; - Ok(FileMetadata { - is_regular: m.is_file(), - size: m.size(), - }) - } - - #[tracing::instrument(skip(self), err)] - async fn get_file_hash( - &self, - path: impl Into + std::fmt::Debug, - ) -> tokio::io::Result { - let file = fs_err::tokio::File::open(path).await?; - let mut reader = BufReader::new(file); - - let mut hasher = Sha256::new(); - let mut buf = [0u8; 16 * 1024]; - - loop { - let n = reader.read(&mut buf).await?; - if n == 0 { - break; - } - hasher.update(&buf[..n]); - } - - Ok(format!("{:x}", hasher.finalize())) - } -} - -fn parse_release_name(content: &str) -> Option { - let content = content.trim(); - if !content.is_empty() && VALIDATE_RELEASE_NAME.is_match(content) { - Some(content.to_owned()) - } else { - None - } -} - -fn parse_metric( - store: &nix_utils::LocalStore, - line: &str, - output: &StorePath, -) -> Option { - let fields: Vec = line.split_whitespace().map(ToOwned::to_owned).collect(); - if fields.len() < 2 || !VALIDATE_METRICS_NAME.is_match(&fields[0]) { - return None; - } - - Some(BuildMetric { - path: store.print_store_path(output), - name: fields[0].clone(), - value: fields[1].parse::().unwrap_or(0.0), - unit: if fields.len() >= 3 && VALIDATE_METRICS_UNIT.is_match(&fields[2]) { - Some(fields[2].clone()) - } else { - None - }, - }) -} - -async fn parse_build_product( - store: &nix_utils::LocalStore, - handle: Op, - output: &StorePath, - line: &str, -) -> Option -where - Op: FsOperations + Send + Sync + 'static, -{ - let captures = BUILD_PRODUCT_PARSER.captures(line)?; - - let s = captures[3].to_string(); - let path = if s.starts_with('"') && s.ends_with('"') { - s[1..s.len() - 1].to_string() - } else { - s - }; - - if path.is_empty() || !path.starts_with('/') { - return None; - } - if !handle.is_inside_store(std::path::Path::new(&path)) { - return None; - } - let metadata = handle.get_metadata(&path).await.ok()?; - - let name = { - let name = if path == store.print_store_path(output) { - String::new() - } else { - std::path::Path::new(&path) - .file_name() - .and_then(|f| f.to_str()) - .map(ToOwned::to_owned) - .unwrap_or_default() - }; - - if VALIDATE_PRODUCT_NAME.is_match(&name) { - name - } else { - String::new() - } - }; - - let sha256hash = if metadata.is_regular { - handle.get_file_hash(&path).await.ok() - } else { - None - }; - - Some(BuildProduct { - r#type: captures[1].to_string(), - subtype: captures[2].to_string(), - path, - default_path: captures - .get(5) - .map(|m| m.as_str().to_string()) - .unwrap_or_default(), - name, - is_regular: metadata.is_regular, - file_size: if metadata.is_regular { - Some(metadata.size) - } else { - None - }, - sha256hash, - }) -} - -#[tracing::instrument(skip(store), err)] -pub async fn parse_nix_support_from_outputs( - store: &nix_utils::LocalStore, - derivation_outputs: &BTreeMap, -) -> anyhow::Result { - let mut metrics = Vec::new(); - let mut failed = false; - let mut hydra_release_name = None; - - for output in derivation_outputs.values() { - let output_full_path = store.print_store_path(output); - let file_path = std::path::Path::new(&output_full_path).join("nix-support/hydra-metrics"); - let Ok(file) = fs_err::tokio::File::open(&file_path).await else { - continue; - }; - - let reader = BufReader::new(file); - let mut lines = reader.lines(); - - while let Some(line) = lines.next_line().await? { - if let Some(m) = parse_metric(store, &line, output) { - metrics.push(m); - } - } - } - - for output in derivation_outputs.values() { - let file_path = - std::path::Path::new(&store.print_store_path(output)).join("nix-support/failed"); - if fs_err::tokio::try_exists(file_path) - .await - .unwrap_or_default() - { - failed = true; - break; - } - } - - for output in derivation_outputs.values() { - let file_path = std::path::Path::new(&store.print_store_path(output)) - .join("nix-support/hydra-release-name"); - if let Ok(v) = fs_err::tokio::read_to_string(file_path).await - && let Some(v) = parse_release_name(&v) - { - hydra_release_name = Some(v); - break; - } - } - - let mut explicit_products = false; - let mut products = Vec::new(); - for output in derivation_outputs.values() { - let output_full_path = store.print_store_path(output); - let file_path = - std::path::Path::new(&output_full_path).join("nix-support/hydra-build-products"); - let Ok(file) = fs_err::tokio::File::open(&file_path).await else { - continue; - }; - - explicit_products = true; - - let reader = BufReader::new(file); - let mut lines = reader.lines(); - let fsop = FilesystemOperations::new(); - while let Some(line) = lines.next_line().await? { - if let Some(o) = Box::pin(parse_build_product(store, fsop.clone(), output, &line)).await - { - products.push(o); - } - } - } - - if !explicit_products { - for (output_name, path) in derivation_outputs { - let full_path = store.print_store_path(path); - let Ok(metadata) = fs_err::tokio::metadata(&full_path).await else { - continue; - }; - if metadata.is_dir() { - products.push(BuildProduct { - r#type: "nix-build".to_string(), - subtype: if output_name.as_ref() == "out" { - String::new() - } else { - output_name.to_string() - }, - path: full_path, - name: path.name().to_string(), - default_path: String::new(), - is_regular: false, - file_size: None, - sha256hash: None, - }); - } - } - } - - Ok(NixSupport { - failed, - hydra_release_name, - metrics, - products, - }) -} - -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used)] - - use super::*; - - #[derive(Debug, Clone)] - struct DummyFsOperations { - valid_file: bool, - metadata: FileMetadata, - file_hash: String, - } - - impl FsOperations for DummyFsOperations { - fn is_inside_store(&self, _: &std::path::Path) -> bool { - self.valid_file - } - - async fn get_metadata( - &self, - _: impl AsRef + std::fmt::Debug, - ) -> Result { - Ok(self.metadata.clone()) - } - - async fn get_file_hash( - &self, - _: impl Into + std::fmt::Debug, - ) -> Result { - Ok(self.file_hash.clone()) - } - } - - #[tokio::test] - async fn test_build_products() { - let store = nix_utils::LocalStore::init(); - let output = nix_utils::parse_store_path("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); - let line = format!( - "file iso {}/iso/custom.iso", - store.print_store_path(&output) - ); - let fsop = DummyFsOperations { - valid_file: true, - metadata: FileMetadata { - is_regular: true, - size: 12345, - }, - file_hash: "4306152c73d2a7a01dbac16ba48f45fa4ae5b746a1d282638524ae2ae93af210".into(), - }; - let build_product = parse_build_product(&store, fsop, &output, &line) - .await - .unwrap(); - assert!(build_product.is_regular); - assert_eq!( - build_product.path, - "/nix/store/ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso/iso/custom.iso" - ); - assert_eq!(build_product.name, "custom.iso"); - assert_eq!(build_product.file_size, Some(12345)); - assert_eq!( - build_product.sha256hash, - Some("4306152c73d2a7a01dbac16ba48f45fa4ae5b746a1d282638524ae2ae93af210".into()) - ); - } - - #[test] - fn test_parse_invalid_metric() { - let output = nix_utils::parse_store_path("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); - let line = "nix-env.qaCount"; - let store = nix_utils::LocalStore::init(); - let m = parse_metric(&store, line, &output); - assert!(m.is_none()); - } - - #[test] - fn test_parse_metric_without_unit() { - let output = nix_utils::parse_store_path("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); - let line = "nix-env.qaCount 4"; - let store = nix_utils::LocalStore::init(); - let m = parse_metric(&store, line, &output).unwrap(); - assert_eq!(m.name, "nix-env.qaCount"); - assert!((m.value - 4.0_f64).abs() < f64::EPSILON); - assert_eq!(m.unit, None); - } - - #[test] - fn test_parse_metric_with_unit() { - let output = nix_utils::parse_store_path("ir3rqjyj5cz3js5lr7d0zw0gn6crzs6w-custom.iso"); - let line = "xzy.time 123.321 s"; - let store = nix_utils::LocalStore::init(); - let m = parse_metric(&store, line, &output).unwrap(); - assert_eq!(m.name, "xzy.time"); - assert!((m.value - 123.321_f64).abs() < f64::EPSILON); - assert_eq!(m.unit, Some("s".into())); - } - - #[test] - fn test_parse_release_name() { - let line = "nixos-25.11pre708350"; - let o = parse_release_name(line); - assert_eq!(o, Some("nixos-25.11pre708350".into())); - } -} diff --git a/subprojects/crates/store-path-utils/Cargo.toml b/subprojects/crates/store-path-utils/Cargo.toml new file mode 100644 index 000000000..1b26ad388 --- /dev/null +++ b/subprojects/crates/store-path-utils/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "store-path-utils" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0" +rust-version.workspace = true + +[dependencies] +harmonia-store-path.workspace = true diff --git a/subprojects/crates/store-path-utils/src/lib.rs b/subprojects/crates/store-path-utils/src/lib.rs new file mode 100644 index 000000000..79dc47e2e --- /dev/null +++ b/subprojects/crates/store-path-utils/src/lib.rs @@ -0,0 +1,112 @@ +use harmonia_store_path::{ParseStorePathError, StoreDir, StorePath}; + +/// A store path with an optional relative sub-path. +/// +/// Represents paths like `/nix/store/-/share/doc/nix/manual`, +/// split into the base `StorePath` (`-`) and the relative +/// suffix (`share/doc/nix/manual`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RelativeStorePath { + pub base_path: StorePath, + pub relative_path: Box, +} + +impl RelativeStorePath { + /// Parse a full filesystem path under a store directory into a + /// `StorePath` and a relative suffix. + /// + /// For `/nix/store/-/foo/bar` returns base=`-`, + /// relative=`"foo/bar"`. + /// For `/nix/store/-` returns base=`-`, + /// relative=`""`. + pub fn from_path(store_dir: &StoreDir, path: &str) -> Result { + let stripped = store_dir + .strip_prefix(path) + .map_err(|e| ParseStorePathError::new(path, e))?; + let (base_str, relative) = stripped.split_once('/').unwrap_or((stripped, "")); + Ok(Self { + base_path: StorePath::from_base_path(base_str)?, + relative_path: relative.into(), + }) + } + + /// Render back into a full filesystem path. + pub fn print(&self, store_dir: &StoreDir) -> String { + if self.relative_path.is_empty() { + store_dir.display(&self.base_path).to_string() + } else { + format!( + "{}/{}", + store_dir.display(&self.base_path), + self.relative_path + ) + } + } +} + +impl From for (StorePath, Box) { + fn from(r: RelativeStorePath) -> Self { + (r.base_path, r.relative_path) + } +} + +impl From<(StorePath, Box)> for RelativeStorePath { + fn from((base_path, relative_path): (StorePath, Box)) -> Self { + Self { + base_path, + relative_path, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_store_dir() -> StoreDir { + StoreDir::new("/nix/store").unwrap() + } + + #[test] + fn splits_product_subpath() { + let dir = test_store_dir(); + let rel = RelativeStorePath::from_path( + &dir, + "/nix/store/bwqqp42xqn37z31dapi7jrhy8iwc2zsx-nix-manual-2.31.4/share/doc/nix/manual", + ) + .expect("subpath product must parse"); + assert_eq!( + rel.base_path.to_string(), + "bwqqp42xqn37z31dapi7jrhy8iwc2zsx-nix-manual-2.31.4" + ); + assert_eq!(&*rel.relative_path, "share/doc/nix/manual"); + } + + #[test] + fn accepts_bare_store_path() { + let dir = test_store_dir(); + let rel = RelativeStorePath::from_path( + &dir, + "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-example-1.0", + ) + .expect("bare store path must parse"); + assert_eq!( + rel.base_path.to_string(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-example-1.0" + ); + assert!(rel.relative_path.is_empty()); + } + + #[test] + fn roundtrips() { + let dir = test_store_dir(); + for original in [ + "/nix/store/bwqqp42xqn37z31dapi7jrhy8iwc2zsx-nix-manual-2.31.4/share/doc/nix/manual", + "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-example-1.0", + ] { + let rel = RelativeStorePath::from_path(&dir, original) + .unwrap_or_else(|e| panic!("parse {original}: {e}")); + assert_eq!(rel.print(&dir), original); + } + } +} diff --git a/subprojects/crates/store-transfer/Cargo.toml b/subprojects/crates/store-transfer/Cargo.toml new file mode 100644 index 000000000..ea11445b4 --- /dev/null +++ b/subprojects/crates/store-transfer/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "store-transfer" +version = "0.1.0" +edition = "2024" +license = "LGPL-2.1-only" +rust-version.workspace = true + +[dependencies] +async-compression = { workspace = true, features = [ "tokio", "zstd" ] } +futures.workspace = true +hashbrown.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = [ "io-util", "rt" ] } +tonic.workspace = true +tracing.workspace = true + +harmonia-protocol.workspace = true +harmonia-store-path.workspace = true +harmonia-store-path-info.workspace = true +harmonia-store-remote.workspace = true +hydra-proto.workspace = true diff --git a/subprojects/crates/store-transfer/src/error.rs b/subprojects/crates/store-transfer/src/error.rs new file mode 100644 index 000000000..2b8a64aca --- /dev/null +++ b/subprojects/crates/store-transfer/src/error.rs @@ -0,0 +1,52 @@ +/// Protocol-level errors in the `AddToStoreRequest` stream. +#[derive(Debug, thiserror::Error)] +pub enum ProtocolError { + #[error("empty AddToStoreRequest stream")] + EmptyStream, + + #[error("expected AddToStoreHeader as first message")] + MissingHeader, + + #[error("PathInfo missing required field: {0}")] + MissingGrpcField(&'static str), + + #[error("invalid path info: {0}")] + InvalidPathInfo(#[from] hydra_proto::NarInfoConvertError), + + #[error("unexpected end of NAR stream for {path}, {remaining} bytes remaining")] + TruncatedNar { + path: harmonia_store_path::StorePath, + remaining: u64, + }, + + #[error("NAR size mismatch for {path}: wrote {actual} bytes, expected {expected}")] + NarSizeMismatch { + path: harmonia_store_path::StorePath, + expected: u64, + actual: u64, + }, +} + +/// Errors during store path import/export over gRPC. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The underlying daemon store operation failed. + #[error(transparent)] + Store(#[from] harmonia_protocol::types::DaemonError), + + /// gRPC transport error. + #[error("gRPC error: {0}")] + Grpc(#[from] tonic::Status), + + /// IO error during NAR streaming. + #[error(transparent)] + Io(#[from] std::io::Error), + + /// The gRPC protocol was violated. + #[error(transparent)] + Protocol(#[from] ProtocolError), + + /// A spawned task panicked or was cancelled. + #[error("task join error: {0}")] + Join(#[from] tokio::task::JoinError), +} diff --git a/subprojects/crates/store-transfer/src/export.rs b/subprojects/crates/store-transfer/src/export.rs new file mode 100644 index 000000000..034a483d5 --- /dev/null +++ b/subprojects/crates/store-transfer/src/export.rs @@ -0,0 +1,117 @@ +//! Shared logic for exporting store paths as an `AddToStoreRequest` stream. + +use harmonia_store_path::StorePath; +use harmonia_store_path_info::UnkeyedValidPathInfo; + +/// Export store paths as `AddToStoreRequest` messages over a gRPC channel. +/// +/// Sends an [`AddToStoreHeader`] containing all path infos first, +/// then zstd-compressed `nar_chunk` messages with all NARs +/// concatenated in the same order. One continuous zstd stream. +/// +/// The `infos` map must contain an entry for every path. Paths +/// missing from the map are skipped. +pub async fn export( + guard: &mut harmonia_store_remote::PooledConnectionGuard, + paths: &[StorePath], + infos: &hashbrown::HashMap, + tx: &tokio::sync::mpsc::UnboundedSender>, +) -> Result<(), crate::Error> { + use tokio::io::AsyncBufReadExt as _; + + // Send header with all path infos (uncompressed). + let proto_infos: Vec = paths + .iter() + .filter_map(|p| { + infos.get(p).map(|info| hydra_proto::ValidPathInfo { + path: Some(hydra_proto::ProtoStorePath::from(p.clone())), + info: Some(hydra_proto::UnkeyedValidPathInfo::from(info)), + }) + }) + .collect(); + + if tx + .send(Ok(hydra_proto::AddToStoreRequest { + content: Some(hydra_proto::add_to_store_request::Content::Header( + hydra_proto::AddToStoreHeader { + path_infos: proto_infos, + }, + )), + })) + .is_err() + { + return Ok(()); + } + + // Stream all NARs as one continuous zstd-compressed stream. + // We use a duplex pipe: write compressed data to one end, + // spawn a task to read from the other and send as gRPC chunks. + let (compressed_writer, compressed_reader) = tokio::io::duplex(crate::PIPE_BUFFER_SIZE); + + let tx_clone = tx.clone(); + let chunk_sender = tokio::spawn(async move { + use tokio::io::AsyncReadExt as _; + let mut reader = compressed_reader; + let mut buf = vec![0u8; crate::COPY_BUFFER_SIZE]; + loop { + let n = reader.read(&mut buf).await?; + if n == 0 { + break; + } + if tx_clone + .send(Ok(hydra_proto::AddToStoreRequest { + content: Some(hydra_proto::add_to_store_request::Content::NarChunk( + buf[..n].to_vec(), + )), + })) + .is_err() + { + break; + } + } + Ok::<(), crate::Error>(()) + }); + + // Write NARs through zstd encoder to the duplex pipe. + let mut encoder = async_compression::tokio::write::ZstdEncoder::new(compressed_writer); + + for path in paths { + let Some(info) = infos.get(path) else { + continue; + }; + + let mut bytes_written: u64 = 0; + use harmonia_protocol::types::DaemonStore; + let mut nar_reader = guard.client().nar_from_path(path).await?; + loop { + let buf = nar_reader.fill_buf().await?; + if buf.is_empty() { + break; + } + use tokio::io::AsyncWriteExt as _; + bytes_written += buf.len() as u64; + encoder.write_all(buf).await?; + let len = buf.len(); + nar_reader.consume(len); + } + + // A mismatch here means the daemon and our recorded path info + // disagree on the NAR contents. + if bytes_written != info.nar_size { + return Err(crate::ProtocolError::NarSizeMismatch { + path: path.clone(), + expected: info.nar_size, + actual: bytes_written, + } + .into()); + } + } + + use tokio::io::AsyncWriteExt as _; + encoder.shutdown().await?; + drop(encoder); + + chunk_sender.await??; + + Ok(()) +} diff --git a/subprojects/crates/store-transfer/src/import.rs b/subprojects/crates/store-transfer/src/import.rs new file mode 100644 index 000000000..59775fd6f --- /dev/null +++ b/subprojects/crates/store-transfer/src/import.rs @@ -0,0 +1,113 @@ +//! Shared logic for importing store paths from an `AddToStoreRequest` stream. + +use crate::{Error, ProtocolError}; + +/// Import store paths from a gRPC `stream AddToStoreRequest`. +/// +/// The first message must be an [`AddToStoreHeader`] containing all +/// [`ValidPathInfo`]s (including NAR sizes). Then zstd-compressed +/// `nar_chunk` bytes with all NARs concatenated in header order. +/// +/// Rust splits the decompressed stream by counting `nar_size` bytes +/// per path and feeds each slice into `add_to_store_nar`. +pub async fn import( + guard: &mut harmonia_store_remote::PooledConnectionGuard, + mut stream: tonic::Streaming, +) -> Result, Error> { + use futures::StreamExt as _; + use harmonia_protocol::types::DaemonStore; + + // First message must be the header. + let first = stream + .next() + .await + .ok_or(ProtocolError::EmptyStream)? + .map_err(Error::Grpc)?; + + let header = match first.content { + Some(hydra_proto::add_to_store_request::Content::Header(h)) => h, + _ => return Err(ProtocolError::MissingHeader.into()), + }; + + let path_infos: Vec = header + .path_infos + .into_iter() + .map(|pi| { + let path = pi.path.ok_or(ProtocolError::MissingGrpcField("path"))?.0; + let info = pi + .info + .ok_or(ProtocolError::MissingGrpcField("info"))? + .try_into() + .map_err(ProtocolError::InvalidPathInfo)?; + Ok(harmonia_store_path_info::ValidPathInfo { path, info }) + }) + .collect::>()?; + + let paths: Vec<_> = path_infos.iter().map(|pi| pi.path.clone()).collect(); + + if path_infos.is_empty() { + return Ok(paths); + } + + // Pipe compressed gRPC chunks through a zstd decoder. A spawned + // task writes the compressed data; we read decompressed bytes + // from the other end. + let (mut compressed_writer, compressed_reader) = tokio::io::duplex(crate::PIPE_BUFFER_SIZE); + + let writer_handle = tokio::spawn(async move { + while let Some(msg) = stream.next().await { + let msg = msg?; + if let Some(hydra_proto::add_to_store_request::Content::NarChunk(chunk)) = msg.content { + use tokio::io::AsyncWriteExt as _; + compressed_writer.write_all(&chunk).await?; + } + } + drop(compressed_writer); + Ok::<(), Error>(()) + }); + + let mut decoder = async_compression::tokio::bufread::ZstdDecoder::new( + tokio::io::BufReader::new(compressed_reader), + ); + + // Import each path by reading exactly nar_size decompressed bytes + // into a per-path duplex pipe feeding add_to_store_nar. + for vpi in &path_infos { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let (mut nar_writer, nar_reader) = tokio::io::duplex(crate::PIPE_BUFFER_SIZE); + let mut nar_buf_reader = tokio::io::BufReader::new(nar_reader); + + let copy_fut = async { + let mut remaining = vpi.info.nar_size; + let mut buf = vec![0u8; crate::COPY_BUFFER_SIZE]; + while remaining > 0 { + let to_read = buf.len().min(remaining as usize); + let n = decoder.read(&mut buf[..to_read]).await?; + if n == 0 { + return Err(ProtocolError::TruncatedNar { + path: vpi.path.clone(), + remaining, + } + .into()); + } + nar_writer.write_all(&buf[..n]).await?; + remaining -= n as u64; + } + drop(nar_writer); + Ok::<(), Error>(()) + }; + + let store_fut = guard + .client() + .add_to_store_nar(vpi, &mut nar_buf_reader, false, true); + + let (copy_result, store_result) = futures::future::join(copy_fut, store_fut).await; + copy_result?; + store_result?; + } + + writer_handle.await??; + + Ok(paths) +} diff --git a/subprojects/crates/store-transfer/src/lib.rs b/subprojects/crates/store-transfer/src/lib.rs new file mode 100644 index 000000000..32b9de6f1 --- /dev/null +++ b/subprojects/crates/store-transfer/src/lib.rs @@ -0,0 +1,13 @@ +mod error; +pub mod export; +pub mod import; + +pub use error::{Error, ProtocolError}; + +/// Capacity of the duplex pipe bridging the zstd codec and the gRPC +/// chunk stream. +const PIPE_BUFFER_SIZE: usize = 256 * 1024; + +/// Size of the scratch buffer used to copy bytes between the duplex pipe +/// and the NAR/chunk streams. +const COPY_BUFFER_SIZE: usize = 64 * 1024; diff --git a/subprojects/crates/tracing/Cargo.toml b/subprojects/crates/tracing/Cargo.toml index 4e675d7f2..3346f8590 100644 --- a/subprojects/crates/tracing/Cargo.toml +++ b/subprojects/crates/tracing/Cargo.toml @@ -6,10 +6,12 @@ license = "GPL-3.0" rust-version.workspace = true [dependencies] -anyhow.workspace = true -tracing.workspace = true -tracing-log.workspace = true -tracing-subscriber = { workspace = true, features = [ "registry", "env-filter" ] } +color-eyre.workspace = true +thiserror.workspace = true +tracing.workspace = true +tracing-error.workspace = true +tracing-log.workspace = true +tracing-subscriber = { workspace = true, features = [ "registry", "env-filter" ] } http = { workspace = true, optional = true } tonic = { workspace = true, optional = true } diff --git a/subprojects/crates/tracing/src/lib.rs b/subprojects/crates/tracing/src/lib.rs index 5faebd46f..dceb304dd 100644 --- a/subprojects/crates/tracing/src/lib.rs +++ b/subprojects/crates/tracing/src/lib.rs @@ -22,6 +22,21 @@ use opentelemetry::trace::TracerProvider as _; #[cfg(feature = "tonic")] pub mod propagate; +#[derive(Debug, thiserror::Error)] +pub enum TracingInitError { + #[error(transparent)] + SetLogger(#[from] tracing_log::log::SetLoggerError), + #[error(transparent)] + SetGlobalDefault(#[from] tracing::subscriber::SetGlobalDefaultError), + // Don't directly include the eyre::Report, if the handler fails to install + // then the traceback would never be printed. + #[error("failed to install color-eyre handler: {0}")] + ColorEyre(String), + #[cfg(feature = "otel")] + #[error(transparent)] + ExporterBuild(#[from] opentelemetry_otlp::ExporterBuildError), +} + #[cfg(feature = "otel")] fn resource() -> opentelemetry_sdk::Resource { opentelemetry_sdk::Resource::builder() @@ -60,7 +75,8 @@ impl Drop for TracingGuard { } #[cfg(feature = "otel")] -fn init_tracer_provider() -> anyhow::Result { +fn init_tracer_provider() +-> Result { let exporter = opentelemetry_otlp::SpanExporter::builder() .with_tonic() .build()?; @@ -71,7 +87,8 @@ fn init_tracer_provider() -> anyhow::Result anyhow::Result { +pub fn init() -> Result { + color_eyre::install().map_err(|e| TracingInitError::ColorEyre(e.to_string()))?; tracing_log::LogTracer::init()?; let (log_env_filter, reload_handle) = tracing_subscriber::reload::Layer::new( EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), @@ -81,7 +98,8 @@ pub fn init() -> anyhow::Result { .compact(); let subscriber = tracing_subscriber::Registry::default() .with(log_env_filter) - .with(fmt_layer); + .with(fmt_layer) + .with(tracing_error::ErrorLayer::default()); #[cfg(feature = "otel")] { diff --git a/subprojects/hydra-builder/Cargo.toml b/subprojects/hydra-builder/Cargo.toml index d5d798ef2..92ef5c319 100644 --- a/subprojects/hydra-builder/Cargo.toml +++ b/subprojects/hydra-builder/Cargo.toml @@ -9,8 +9,8 @@ rust-version.workspace = true sd-notify.workspace = true tracing.workspace = true -anyhow.workspace = true clap = { workspace = true, features = [ "derive" ] } +color-eyre.workspace = true fs-err = { workspace = true, features = [ "tokio" ] } hashbrown.workspace = true parking_lot.workspace = true @@ -22,6 +22,7 @@ async-stream.workspace = true backon.workspace = true bytes.workspace = true futures.workspace = true +http.workspace = true hyper-util.workspace = true prost.workspace = true tokio = { workspace = true, features = [ "full" ] } @@ -35,16 +36,24 @@ url.workspace = true serde.workspace = true serde_json.workspace = true -harmonia-store-core.workspace = true +harmonia-protocol.workspace = true +harmonia-store-derivation.workspace = true +harmonia-store-path.workspace = true +harmonia-store-remote.workspace = true +harmonia-utils-hash.workspace = true gethostname.workspace = true nix = { workspace = true, features = [ "fs" ] } procfs-core.workspace = true -binary-cache = { path = "../crates/binary-cache" } -hydra-tracing = { path = "../crates/tracing", features = [ "tonic" ] } -nix-utils = { path = "../crates/nix-utils" } -shared = { path = "../crates/shared" } +binary-cache.workspace = true +daemon-client-utils.workspace = true +harmonia-store-path-info.workspace = true +hydra-proto = { workspace = true, features = [ "client" ] } +hydra-tracing = { workspace = true, features = [ "tonic" ] } +nix-support.workspace = true +store-path-utils.workspace = true +store-transfer.workspace = true [target.'cfg(target_os = "macos")'.dependencies] sysinfo.workspace = true @@ -52,10 +61,5 @@ sysinfo.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator.workspace = true -[build-dependencies] -fs-err = { workspace = true } -sha2.workspace = true -tonic-prost-build.workspace = true - [features] otel = [ "hydra-tracing/otel" ] diff --git a/subprojects/hydra-builder/build.rs b/subprojects/hydra-builder/build.rs deleted file mode 100644 index c03299460..000000000 --- a/subprojects/hydra-builder/build.rs +++ /dev/null @@ -1,31 +0,0 @@ -use sha2::Digest; -use std::{env, path::PathBuf}; - -fn main() -> Result<(), Box> { - let out_dir = PathBuf::from(env::var("OUT_DIR")?); - - let workspace_version = env::var("CARGO_PKG_VERSION")?; - - let proto_path = "../proto/v1/streaming.proto"; - let proto_content = fs_err::read_to_string(proto_path)?; - let mut hasher = sha2::Sha256::new(); - hasher.update(proto_content.as_bytes()); - let proto_hash = format!("{:x}", hasher.finalize()); - let version = format!("{}-{}", workspace_version, &proto_hash[..8]); - - fs_err::write( - out_dir.join("proto_version.rs"), - format!( - r#"// Generated during build - do not edit -pub const PROTO_API_VERSION: &str = "{version}"; -"# - ), - )?; - - tonic_prost_build::configure() - .extern_path(".runner.v1.StorePath", "::shared::proto::ProtoStorePath") - .build_server(false) - .file_descriptor_set_path(out_dir.join("streaming_descriptor.bin")) - .compile_protos(&["../proto/v1/streaming.proto"], &["../proto"])?; - Ok(()) -} diff --git a/subprojects/hydra-builder/package.nix b/subprojects/hydra-builder/package.nix index 1dafe32b6..c2471eea3 100644 --- a/subprojects/hydra-builder/package.nix +++ b/subprojects/hydra-builder/package.nix @@ -4,7 +4,6 @@ rustPlatform, - nixComponents, protobuf, pkg-config, rust-jemalloc-sys, @@ -22,7 +21,6 @@ rustPlatform.buildRustPackage { ../../Cargo.lock ../../.cargo ../../subprojects/hydra-builder/Cargo.toml - ../../subprojects/hydra-builder/build.rs ../../subprojects/hydra-builder/src ../../subprojects/crates # For unit tests which want to spin up a fresh database @@ -33,9 +31,7 @@ rustPlatform.buildRustPackage { cargoLock = { lockFile = ../../Cargo.lock; - outputHashes = { - "harmonia-store-core-0.0.0-alpha.0" = "sha256-T6Mbhet2sNGqU9wT5keCAKCSJKrDJ1NuuvtmWp7XUPY="; - }; + outputHashes = import ../../packaging/cargo-output-hashes.nix; }; # The source fileset above intentionally excludes hydra-queue-runner, @@ -54,7 +50,6 @@ rustPlatform.buildRustPackage { ]; buildInputs = [ - nixComponents.nix-main protobuf rust-jemalloc-sys ]; diff --git a/subprojects/hydra-builder/src/config.rs b/subprojects/hydra-builder/src/config.rs index 04792270e..3ffed3a59 100644 --- a/subprojects/hydra-builder/src/config.rs +++ b/subprojects/hydra-builder/src/config.rs @@ -1,5 +1,21 @@ use clap::Parser; +/// Errors from reading builder configuration (mTLS certs, etc.). +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("missing config option: {0}")] + MissingOption(&'static str), + + #[error("Reading configuration file")] + Reading(#[source] std::io::Error), + + #[error( + "mTLS configured improperly, please pass all options: \ + server_root_ca_cert_path, client_cert_path, client_key_path and domain_name" + )] + MtlsIncomplete, +} + #[derive(Parser, Debug)] #[clap( author, @@ -117,44 +133,54 @@ impl Cli { #[tracing::instrument(skip(self), err)] pub async fn get_mtls( &self, - ) -> anyhow::Result<( - tonic::transport::Certificate, - tonic::transport::Identity, - String, - )> { + ) -> Result< + ( + tonic::transport::Certificate, + tonic::transport::Identity, + String, + ), + ConfigError, + > { let server_root_ca_cert_path = self .server_root_ca_cert_path .as_deref() - .ok_or_else(|| anyhow::anyhow!("server_root_ca_cert_path not provided"))?; + .ok_or(ConfigError::MissingOption("server_root_ca_cert_path"))?; let client_cert_path = self .client_cert_path .as_deref() - .ok_or_else(|| anyhow::anyhow!("client_cert_path not provided"))?; + .ok_or(ConfigError::MissingOption("client_cert_path"))?; let client_key_path = self .client_key_path .as_deref() - .ok_or_else(|| anyhow::anyhow!("client_key_path not provided"))?; + .ok_or(ConfigError::MissingOption("client_key_path"))?; let domain_name = self .domain_name .as_deref() - .ok_or_else(|| anyhow::anyhow!("domain_name not provided"))?; + .ok_or(ConfigError::MissingOption("domain_name"))?; - let server_root_ca_cert = fs_err::tokio::read_to_string(server_root_ca_cert_path).await?; + let server_root_ca_cert = fs_err::tokio::read_to_string(server_root_ca_cert_path) + .await + .map_err(ConfigError::Reading)?; let server_root_ca_cert = tonic::transport::Certificate::from_pem(server_root_ca_cert); - let client_cert = fs_err::tokio::read_to_string(client_cert_path).await?; - let client_key = fs_err::tokio::read_to_string(client_key_path).await?; + let client_cert = fs_err::tokio::read_to_string(client_cert_path) + .await + .map_err(ConfigError::Reading)?; + let client_key = fs_err::tokio::read_to_string(client_key_path) + .await + .map_err(ConfigError::Reading)?; let client_identity = tonic::transport::Identity::from_pem(client_cert, client_key); Ok((server_root_ca_cert, client_identity, domain_name.to_owned())) } #[tracing::instrument(skip(self), err)] - pub async fn get_authorization_token(&self) -> anyhow::Result> { + pub async fn get_authorization_token(&self) -> Result, ConfigError> { if let Some(path) = &self.authorization_file { Ok(Some( fs_err::tokio::read_to_string(path) - .await? + .await + .map_err(ConfigError::Reading)? .trim() .to_string(), )) diff --git a/subprojects/hydra-builder/src/error.rs b/subprojects/hydra-builder/src/error.rs new file mode 100644 index 000000000..81d2720b5 --- /dev/null +++ b/subprojects/hydra-builder/src/error.rs @@ -0,0 +1,58 @@ +use color_eyre::eyre; + +#[derive(Debug, thiserror::Error)] +pub enum BuilderError { + #[error("environment variable {0} not set")] + MissingEnvVar(&'static str), + + #[error("creating gcroots directory")] + CreateGcroots(#[source] std::io::Error), + + #[error("hostname is not valid UTF-8: {0:?}")] + Hostname(std::ffi::OsString), + + #[error("Requesting presigned URLs")] + PresignedUrls(#[source] tonic::Status), + + #[error("Parsing configuration")] + Configuration(#[from] crate::config::ConfigError), + + #[error("Parsing gateway endpoint")] + GatewayEndpoint(#[source] http::uri::InvalidUri), + + #[error("Parsing Nix store URL: {0}")] + ParseNixStore(String), + + #[error("Loading Nix configuration")] + LoadNixConfig(#[source] eyre::Report), + + #[error("Gateway API missing host")] + GatewayMissingHost, + + #[error("Connecting to channel")] + Connection(#[source] tonic::transport::Error), + + #[error("Attaching TLS Configuration")] + TlsConfig(#[source] tonic::transport::Error), + + #[error("Incorrectly formatted authorisation token")] + AuthToken(#[source] tonic::metadata::errors::InvalidMetadataValue), + + #[error("Calling service")] + CallingService(#[source] tonic::Status), + + #[error("API version mismatch: client has {ver}, server has {0}", ver=hydra_proto::PROTO_API_VERSION)] + VersionIncompatible(String), + + #[error("Reading system information")] + ReadingSystemInfo(#[source] eyre::Report), + + #[error("Failed to communicate {0} times over the channel. Terminating the application.")] + RepeatedFailure(u32), + + #[error("While handling request")] + HandlingRequest(#[source] eyre::Report), + + #[error("Task failed")] + Task(#[from] tokio::task::JoinError), +} diff --git a/subprojects/hydra-builder/src/grpc.rs b/subprojects/hydra-builder/src/grpc.rs index 9b07abfe9..970645844 100644 --- a/subprojects/hydra-builder/src/grpc.rs +++ b/subprojects/hydra-builder/src/grpc.rs @@ -1,21 +1,15 @@ +use crate::error::BuilderError; use std::sync::Arc; use std::sync::atomic::Ordering; -use anyhow::Context as _; use tonic::{Request, service::interceptor::InterceptedService, transport::Channel}; -use runner_v1::{ +use harmonia_store_path::StorePath; +use hydra_proto::{ BuilderRequest, VersionCheckRequest, builder_request, runner_request, runner_service_client::RunnerServiceClient, }; -pub mod runner_v1 { - // We need to allow pedantic here because of generated code - #![allow(clippy::pedantic, unused_qualifications)] - - tonic::include_proto!("runner.v1"); -} - #[derive(Debug, Clone)] pub enum BuilderInterceptor { Token { @@ -38,7 +32,21 @@ impl tonic::service::Interceptor for BuilderInterceptor { } } -pub type BuilderClient = RunnerServiceClient>; +#[derive(Debug, Clone)] +pub struct BuilderClient(pub RunnerServiceClient>); + +impl std::ops::Deref for BuilderClient { + type Target = RunnerServiceClient>; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for BuilderClient { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} impl BuilderClient { #[tracing::instrument(skip(self, store_paths), err)] @@ -46,16 +54,19 @@ impl BuilderClient { &mut self, build_id: &str, machine_id: &str, - store_paths: Vec<(nix_utils::StorePath, String, Vec)>, - ) -> anyhow::Result> { - use runner_v1::{PresignedNarRequest, PresignedUrlRequest}; + store_paths: Vec<(StorePath, harmonia_store_path_info::NarHash, Vec)>, + ) -> Result, BuilderError> { + use hydra_proto::{PresignedNarRequest, PresignedUrlRequest}; let request = store_paths .into_iter() - .map(|(path, nar_hash, build_ids)| PresignedNarRequest { - store_path: path.to_string(), - nar_hash, - debug_info_build_ids: build_ids, + .map(|(path, nar_hash, build_ids)| { + let hash: harmonia_utils_hash::Hash = nar_hash.into(); + PresignedNarRequest { + store_path: path.to_string(), + nar_hash: Some((&hash).into()), + debug_info_build_ids: build_ids, + } }) .collect::>(); @@ -66,43 +77,47 @@ impl BuilderClient { request, }) .await - .context("Failed to request presigned URLs")?; + .map_err(BuilderError::PresignedUrls)?; Ok(response.into_inner().inner) } } #[tracing::instrument(err)] -pub async fn init_client(cli: &crate::config::Cli) -> anyhow::Result { +pub async fn init_client(cli: &crate::config::Cli) -> Result { if !cli.mtls_configured_correctly() { tracing::error!( "mtls configured inproperly, please pass all options: \ server_root_ca_cert_path, client_cert_path, client_key_path and domain_name!" ); - return Err(anyhow::anyhow!("Configuration issue")); + return Err(BuilderError::Configuration( + crate::config::ConfigError::MtlsIncomplete, + )); } tracing::info!("connecting to {}", cli.gateway_endpoint); let channel = if cli.mtls_enabled() { tracing::info!("mtls is enabled"); - let (server_root_ca_cert, client_identity, domain_name) = cli - .get_mtls() - .await - .context("Failed to get_mtls Certificate and Identity")?; + let (server_root_ca_cert, client_identity, domain_name) = + cli.get_mtls().await.map_err(BuilderError::Configuration)?; let tls = tonic::transport::ClientTlsConfig::new() .domain_name(domain_name) .ca_certificate(server_root_ca_cert) .identity(client_identity); - Channel::builder(cli.gateway_endpoint.parse()?) - .tls_config(tls) - .context("Failed to attach tls config")? - .connect() - .await - .context("Failed to establish connection with Channel")? + Channel::builder( + cli.gateway_endpoint + .parse() + .map_err(BuilderError::GatewayEndpoint)?, + ) + .tls_config(tls) + .map_err(BuilderError::TlsConfig)? + .connect() + .await + .map_err(BuilderError::Connection)? } else if let Some(path) = cli.gateway_endpoint.strip_prefix("unix://") { let path = path.to_owned(); - tonic::transport::Endpoint::try_from("http://[::]:50051")? + tonic::transport::Endpoint::from_static("http://[::]:50051") .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { let path = path.clone(); async move { @@ -112,50 +127,59 @@ pub async fn init_client(cli: &crate::config::Cli) -> anyhow::Result, request: runner_request::Message, -) -> anyhow::Result<()> { +) -> Result<(), BuilderError> { match request { runner_request::Message::Join(m) => { state @@ -169,52 +193,58 @@ async fn handle_request( } runner_request::Message::Ping(_) => (), runner_request::Message::Build(m) => { - state.schedule_build(m)?; + state + .schedule_build(m) + .map_err(BuilderError::HandlingRequest)?; } runner_request::Message::Abort(m) => { - state.abort_build(&m)?; + state + .abort_build(&m) + .map_err(BuilderError::HandlingRequest)?; } } Ok(()) } #[tracing::instrument(skip(state), err)] -async fn check_version_compatibility(state: Arc) -> anyhow::Result<()> { +async fn check_version_compatibility(state: Arc) -> Result<(), BuilderError> { let mut client = state.client.clone(); let response = client .check_version(Request::new(VersionCheckRequest { - version: crate::state::PROTO_API_VERSION.to_string(), + version: hydra_proto::PROTO_API_VERSION.to_string(), machine_id: state.id.to_string(), hostname: state.hostname.clone(), - store_dir: nix_utils::get_store_dir().to_string(), + store_dir: state.config.store_dir.to_string(), })) - .await?; + .await + .map_err(BuilderError::CallingService)?; let response = response.into_inner(); if !response.compatible { - return Err(anyhow::anyhow!( - "API version mismatch: client has {}, server has {}", - crate::state::PROTO_API_VERSION, - response.server_version, - )); + return Err(BuilderError::VersionIncompatible(response.server_version)); } tracing::info!( "Version check passed: client={}, server={}", - crate::state::PROTO_API_VERSION, + hydra_proto::PROTO_API_VERSION, response.server_version ); Ok(()) } #[tracing::instrument(skip(state), err)] -pub async fn start_bidirectional_stream(state: Arc) -> anyhow::Result<()> { +pub async fn start_bidirectional_stream( + state: Arc, +) -> Result<(), BuilderError> { use tokio_stream::StreamExt as _; check_version_compatibility(state.clone()).await?; - let join_msg = state.get_join_message().await?; + let join_msg = state + .get_join_message() + .await + .map_err(BuilderError::ReadingSystemInfo)?; let state2 = state.clone(); let ping_stream = async_stream::stream! { yield BuilderRequest { @@ -249,11 +279,7 @@ pub async fn start_bidirectional_stream(state: Arc) -> anyh let mut stream = match response { Ok(response) => response.into_inner(), Err(e) => { - let error_str = e.to_string(); - if error_str.contains("API version mismatch") { - return Err(anyhow::anyhow!("API version mismatch: {error_str}")); - } - return Err(e.into()); + return Err(BuilderError::CallingService(e)); } }; @@ -273,10 +299,7 @@ pub async fn start_bidirectional_stream(state: Arc) -> anyh consecutive_failure_count += 1; tracing::error!("stream message delivery failed: {e}"); if consecutive_failure_count == 10 { - return Err(anyhow::anyhow!( - "Failed to communicate {consecutive_failure_count} times over the channel. \ - Terminating the application." - )); + return Err(BuilderError::RepeatedFailure(consecutive_failure_count)); } } } diff --git a/subprojects/hydra-builder/src/lib.rs b/subprojects/hydra-builder/src/lib.rs index 269e65392..d75a53cd3 100644 --- a/subprojects/hydra-builder/src/lib.rs +++ b/subprojects/hydra-builder/src/lib.rs @@ -14,9 +14,11 @@ #![allow(clippy::missing_errors_doc)] pub mod config; +pub mod error; pub mod grpc; pub mod metrics; +pub mod nix_config; pub mod state; pub mod system; pub mod types; -pub mod utils; +mod utils; diff --git a/subprojects/hydra-builder/src/main.rs b/subprojects/hydra-builder/src/main.rs index 7977fc281..3fddca69b 100644 --- a/subprojects/hydra-builder/src/main.rs +++ b/subprojects/hydra-builder/src/main.rs @@ -12,9 +12,13 @@ )] #![allow(clippy::missing_errors_doc)] +use crate::error::BuilderError; + mod config; +mod error; mod grpc; mod metrics; +mod nix_config; mod state; mod system; mod types; @@ -40,9 +44,8 @@ async fn stop_application( } #[tokio::main] -async fn main() -> anyhow::Result<()> { +async fn main() -> color_eyre::Result<()> { let _tracing_guard = hydra_tracing::init()?; - nix_utils::init_nix(); let cli = config::Cli::new(); @@ -66,27 +69,25 @@ async fn main() -> anyhow::Result<()> { _ = sigint.recv() => { tracing::info!("Received sigint - shutting down gracefully"); stop_application(&state, &abort_handle).await; + Ok(()) } _ = sigterm.recv() => { tracing::info!("Received sigterm - shutting down gracefully"); stop_application(&state, &abort_handle).await; + Ok(()) } r = task => { let _ = state.clear_gcroots().await; - match r { - Ok(Ok(())) => (), - Ok(Err(e)) => { - let error_str = e.to_string(); - if error_str.contains("API version mismatch") { - tracing::error!("ERROR: {error_str}"); - std::process::exit(65); // EX_DATAERR - } else { - return Err(e); - } + match r.map_err(BuilderError::from).flatten() { + Ok(()) => Ok(()), + Err(e) => match e { + BuilderError::VersionIncompatible(_) => { + tracing::error!("ERROR: {e:?}"); + std::process::exit(65) // EX_DATAERR + }, + _=> Err(e.into()) } - Err(e) => return Err(e.into()), } } - }; - Ok(()) + } } diff --git a/subprojects/hydra-builder/src/nix_config.rs b/subprojects/hydra-builder/src/nix_config.rs new file mode 100644 index 000000000..8eb18ee9f --- /dev/null +++ b/subprojects/hydra-builder/src/nix_config.rs @@ -0,0 +1,105 @@ +//! Read nix configuration by shelling out to `nix show-config --json`. + +use color_eyre::eyre; +use std::collections::HashMap; + +/// Cached nix configuration values. +#[derive(Debug, Clone)] +pub struct NixConfig { + values: HashMap, +} + +impl NixConfig { + /// Read nix configuration by running `nix show-config --json`. + pub fn load() -> eyre::Result { + let output = std::process::Command::new("nix") + .args([ + "--extra-experimental-features", + "nix-command", + "show-config", + "--json", + ]) + .output()?; + if !output.status.success() { + eyre::bail!( + "nix show-config failed: {}", + str::from_utf8(&output.stderr).unwrap_or("Invalid UTF-8") + ); + } + let values: HashMap = serde_json::from_slice(&output.stdout)?; + Ok(Self { values }) + } + + fn get_string(&self, key: &str) -> Option { + self.values + .get(key)? + .get("value")? + .as_str() + .map(String::from) + } + + fn get_string_list(&self, key: &str) -> Vec { + self.values + .get(key) + .and_then(|v| v.get("value")) + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default() + } + + fn get_bool(&self, key: &str) -> bool { + self.values + .get(key) + .and_then(|v| v.get("value")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + } + + #[must_use] + pub fn system(&self) -> String { + self.get_string("system") + .unwrap_or_else(|| std::env::consts::ARCH.to_owned() + "-linux") + } + + #[must_use] + pub fn extra_platforms(&self) -> Vec { + self.get_string_list("extra-platforms") + } + + #[must_use] + pub fn system_features(&self) -> Vec { + self.get_string_list("system-features") + } + + #[must_use] + pub fn substituters(&self) -> Vec { + self.get_string_list("substituters") + } + + #[must_use] + pub fn use_cgroups(&self) -> bool { + self.get_bool("use-cgroups") + } + + #[must_use] + pub fn build_dir(&self) -> String { + self.get_string("build-dir") + .unwrap_or_else(|| "/tmp".to_owned()) + } + + #[must_use] + #[allow(clippy::unused_self)] + pub fn nix_version(&self) -> String { + std::process::Command::new("nix") + .arg("--version") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_owned()) + .unwrap_or_default() + } +} diff --git a/subprojects/hydra-builder/src/state.rs b/subprojects/hydra-builder/src/state.rs index e2944a02b..30ef3e252 100644 --- a/subprojects/hydra-builder/src/state.rs +++ b/subprojects/hydra-builder/src/state.rs @@ -3,26 +3,24 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::time::Instant; -use anyhow::Context as _; use backon::RetryableWithContext as _; -use binary_cache::harmonia_utils_hash::fmt::CommonHash as _; +use color_eyre::eyre::{self, WrapErr as _}; use futures::TryFutureExt as _; +use harmonia_protocol::daemon_wire::types2::BuildResultSuccess; +use harmonia_store_remote::DaemonStore as _; use hashbrown::HashMap; -use tonic::Request; -use crate::grpc::{BuilderClient, runner_v1}; +use crate::error::BuilderError; +use crate::grpc::BuilderClient; use crate::types::BuildTimings; use binary_cache::{Compression, PresignedUpload, PresignedUploadClient}; -use nix_utils::{BaseStore as _, OutputName}; -use runner_v1::{ - AbortMessage, BuildMessage, BuildMetric, BuildProduct, BuildResultInfo, BuildResultState, - FetchRequisitesRequest, JoinMessage, NarData, NixSupport, Output, OutputNameOnly, - OutputWithPath, PingMessage, PressureState, StepStatus, StepUpdate, StorePaths, output, +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_path::StorePath; +use hydra_proto::ProtoStorePath; +use hydra_proto::{ + AbortMessage, BuildMessage, BuildResultInfo, BuildResultState, JoinMessage, OutputInfo, + PingMessage, StepStatus, StepUpdate, }; -use shared::proto::ProtoStorePath; - -include!(concat!(env!("OUT_DIR"), "/proto_version.rs")); - const RETRY_MIN_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(3); const RETRY_MAX_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(90); @@ -35,27 +33,20 @@ fn retry_strategy() -> backon::ExponentialBuilder { #[derive(thiserror::Error, Debug)] pub enum JobFailure { - #[error("Build failure: `{0}`")] - Build(anyhow::Error), - #[error("Preparing failure: `{0}`")] - Preparing(anyhow::Error), - #[error("Import failure: `{0}`")] - Import(anyhow::Error), - #[error("Upload failure: `{0}`")] - Upload(anyhow::Error), - #[error("Post processing failure: `{0}`")] - PostProcessing(anyhow::Error), + #[error("Build failure")] + Build(#[source] eyre::Report), + #[error("Preparing failure")] + Preparing(#[source] eyre::Report), + #[error("Import failure")] + Import(#[source] eyre::Report), + #[error("Upload failure")] + Upload(#[source] eyre::Report), + #[error("Post processing failure")] + PostProcessing(#[source] eyre::Report), } -#[derive(Debug, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct NixBuildOutputs { - drv_path: String, - outputs: BTreeMap, -} - -impl From for BuildResultState { - fn from(item: JobFailure) -> Self { +impl From<&JobFailure> for BuildResultState { + fn from(item: &JobFailure) -> Self { match item { JobFailure::Build(_) => Self::BuildFailure, JobFailure::Preparing(_) => Self::PreparingFailure, @@ -68,7 +59,7 @@ impl From for BuildResultState { #[derive(Debug)] pub struct BuildInfo { - drv_path: nix_utils::StorePath, + drv_path: StorePath, handle: tokio::task::JoinHandle<()>, was_cancelled: Arc, } @@ -97,14 +88,22 @@ pub struct Config { pub mandatory_features: Vec, pub cgroups: bool, pub use_substitutes: bool, + pub substituters: Vec, + pub nix_version: String, + pub build_dir: String, + pub store_dir: harmonia_store_path::StoreDir, + /// Physical store directory on disk (for chroot stores). + /// `None` means the logical store dir is the filesystem path. + pub real_store_dir: Option, } -#[derive(Debug)] +#[allow(missing_debug_implementations)] pub struct State { pub id: uuid::Uuid, pub hostname: String, pub config: Config, pub max_concurrent_downloads: AtomicU32, + pub pool: harmonia_store_remote::ConnectionPool, active_builds: parking_lot::RwLock>>, pub client: BuilderClient, @@ -141,26 +140,34 @@ impl Drop for Gcroot { impl State { #[tracing::instrument(err)] - pub async fn new(cli: &super::config::Cli) -> anyhow::Result> { - nix_utils::set_verbosity(1); - - let logname = std::env::var("LOGNAME").context("LOGNAME not set")?; - let nix_state_dir = - std::env::var("NIX_STATE_DIR").unwrap_or_else(|_| "/nix/var/nix/".to_owned()); - let gcroots = std::path::PathBuf::from(nix_state_dir) + pub async fn new(cli: &super::config::Cli) -> Result, BuilderError> { + let nix_config = + crate::nix_config::NixConfig::load().map_err(BuilderError::LoadNixConfig)?; + let nix_remote = + daemon_client_utils::parse_nix_remote().map_err(BuilderError::ParseNixStore)?; + + let logname = + std::env::var("LOGNAME").map_err(|_| BuilderError::MissingEnvVar("LOGNAME"))?; + let gcroots = nix_remote + .state_dir .join("gcroots/per-user") .join(logname) .join("hydra-roots/builder"); - fs_err::tokio::create_dir_all(&gcroots).await?; + fs_err::tokio::create_dir_all(&gcroots) + .await + .map_err(BuilderError::CreateGcroots)?; + + let pool = harmonia_store_remote::ConnectionPool::with_store_dir( + &nix_remote.socket, + nix_remote.store_dir.clone(), + harmonia_store_remote::PoolConfig::default(), + ); let state = Arc::new(Self { id: uuid::Uuid::new_v4(), - hostname: gethostname::gethostname().into_string().map_err(|v| { - anyhow::anyhow!( - "Couldn't convert hostname to string! OsString={}", - v.display() - ) - })?, + hostname: gethostname::gethostname() + .into_string() + .map_err(BuilderError::Hostname)?, active_builds: parking_lot::RwLock::new(HashMap::with_capacity(10)), config: Config { ping_interval: cli.ping_interval, @@ -176,8 +183,8 @@ impl State { systems: cli.systems.as_ref().map_or_else( || { let mut out = Vec::with_capacity(8); - out.push(nix_utils::get_this_system()); - out.extend(nix_utils::get_extra_platforms()); + out.push(nix_config.system()); + out.extend(nix_config.extra_platforms()); out }, Clone::clone, @@ -185,11 +192,17 @@ impl State { supported_features: cli .supported_features .as_ref() - .map_or_else(nix_utils::get_system_features, Clone::clone), + .map_or_else(|| nix_config.system_features(), Clone::clone), mandatory_features: cli.mandatory_features.clone().unwrap_or_default(), - cgroups: nix_utils::get_use_cgroups(), + cgroups: nix_config.use_cgroups(), use_substitutes: cli.use_substitutes, + substituters: nix_config.substituters(), + nix_version: nix_config.nix_version(), + build_dir: nix_config.build_dir(), + store_dir: nix_remote.store_dir.clone(), + real_store_dir: nix_remote.real_store_dir(), }, + pool, max_concurrent_downloads: 5.into(), client: crate::grpc::init_client(cli).await?, halt: false.into(), @@ -211,7 +224,7 @@ impl State { } #[tracing::instrument(skip(self), err)] - pub async fn get_join_message(&self) -> anyhow::Result { + pub async fn get_join_message(&self) -> eyre::Result { let sys = crate::system::BaseSystemInfo::new()?; Ok(JoinMessage { @@ -232,15 +245,23 @@ impl State { supported_features: self.config.supported_features.clone(), mandatory_features: self.config.mandatory_features.clone(), cgroups: self.config.cgroups, - substituters: nix_utils::get_substituters(), + substituters: self.config.substituters.clone(), use_substitutes: self.config.use_substitutes, - nix_version: nix_utils::get_nix_version(), + nix_version: self.config.nix_version.clone(), }) } #[tracing::instrument(skip(self), err)] - pub fn get_ping_message(&self) -> anyhow::Result { - let sysinfo = crate::system::SystemLoad::new(&nix_utils::get_build_dir())?; + pub fn get_ping_message(&self) -> eyre::Result { + let default_store = self.config.store_dir.to_string(); + let store_path = self + .config + .real_store_dir + .as_ref() + .map_or(default_store.as_str(), |p| { + p.to_str().unwrap_or(default_store.as_str()) + }); + let sysinfo = crate::system::SystemLoad::new(&self.config.build_dir, store_path)?; Ok(PingMessage { machine_id: self.id.to_string(), @@ -248,14 +269,7 @@ impl State { load5: sysinfo.load_avg_5, load15: sysinfo.load_avg_15, mem_usage: sysinfo.mem_usage, - pressure: sysinfo.pressure.map(|p| PressureState { - cpu_some: p.cpu_some.map(Into::into), - mem_some: p.mem_some.map(Into::into), - mem_full: p.mem_full.map(Into::into), - io_some: p.io_some.map(Into::into), - io_full: p.io_full.map(Into::into), - irq_full: p.irq_full.map(Into::into), - }), + pressure: sysinfo.pressure, build_dir_free_percent: sysinfo.build_dir_free_percent, store_free_percent: sysinfo.store_free_percent, current_substituting_path_count: self.metrics.get_substituting_path_count(), @@ -265,17 +279,13 @@ impl State { } #[tracing::instrument(skip(self, m), fields(drv=?m.drv))] - pub fn schedule_build(self: Arc, m: BuildMessage) -> anyhow::Result<()> { + pub fn schedule_build(self: Arc, m: BuildMessage) -> eyre::Result<()> { if self.halt.load(Ordering::SeqCst) { tracing::warn!("State is set to halt, will no longer accept new builds!"); - return Err(anyhow::anyhow!("State set to halt.")); + return Err(eyre::eyre!("State set to halt.")); } - let drv = m - .drv - .clone() - .ok_or_else(|| anyhow::anyhow!("missing drv"))? - .0; + let drv = m.drv.clone().ok_or_else(|| eyre::eyre!("missing drv"))?.0; if self.contains_build(&drv) { return Ok(()); } @@ -297,12 +307,15 @@ impl State { Err(e) => { if was_cancelled.load(Ordering::SeqCst) { tracing::error!( - "Build of {drv} was cancelled {e}, not reporting Error" + "Build of {drv} was cancelled, not reporting Error: {:?}", + eyre::Report::new(e) ); return; } - tracing::error!("Build of {drv} failed with {e}"); + let result_state = BuildResultState::from(&e) as i32; + + tracing::error!("Build of {drv} failed with: {:?}", eyre::Report::new(e)); self_.remove_build(build_id); let failed_build = BuildResultInfo { build_id: build_id.to_string(), @@ -313,9 +326,8 @@ impl State { .unwrap_or_default(), upload_time_ms: u64::try_from(timings.upload_elapsed.as_millis()) .unwrap_or_default(), - result_state: BuildResultState::from(e) as i32, - nix_support: None, - outputs: vec![], + result_state, + output_infos: std::collections::HashMap::new(), }; if let (_, Err(e)) = (|tuple: (BuilderClient, BuildResultInfo)| async { @@ -349,7 +361,7 @@ impl State { Ok(()) } - fn contains_build(&self, drv: &nix_utils::StorePath) -> bool { + fn contains_build(&self, drv: &StorePath) -> bool { let active = self.active_builds.read(); active.values().any(|b| b.drv_path == *drv) } @@ -372,7 +384,7 @@ impl State { } #[tracing::instrument(skip(self, m), fields(build_id=%m.build_id))] - pub fn abort_build(&self, m: &AbortMessage) -> anyhow::Result<()> { + pub fn abort_build(&self, m: &AbortMessage) -> eyre::Result<()> { tracing::info!("Try cancelling build"); let build_id = uuid::Uuid::parse_str(&m.build_id)?; if let Some(b) = self.remove_build(build_id) { @@ -389,6 +401,104 @@ impl State { active.clear(); } + #[tracing::instrument(skip(self, pool, basic_drv))] + async fn request_build( + &self, + pool: &harmonia_store_remote::ConnectionPool, + drv: &StorePath, + basic_drv: &harmonia_store_derivation::derivation::BasicDerivation, + max_log_size: u64, + max_silent_time: i32, + build_timeout: i32, + ) -> eyre::Result { + // Build the pre-resolved BasicDerivation; logs stream to the queue-runner. + let mut guard = pool.acquire().await.wrap_err("daemon connection failed")?; + + let (log_tx, log_rx) = tokio::sync::mpsc::unbounded_channel::(); + let log_stream = crate::utils::compressed_log_stream(drv, log_rx); + let log_handle = tokio::spawn({ + let mut client = self.client.clone(); + async move { client.build_log(log_stream).await } + }); + + let build_result = { + use futures::stream::StreamExt as _; + + { + let mut options = harmonia_protocol::types::ClientOptions::default(); + options.max_silent_time = i64::from(max_silent_time); + options + .other_settings + .insert("max-log-size".to_string(), max_log_size.to_string().into()); + options + .other_settings + .insert("timeout".to_string(), build_timeout.to_string().into()); + // build_derivation does not take options; they must be + // set per-client with a separate call. + guard.client().set_options(&options).await?; + } + + let mut result_log = + std::pin::pin!(harmonia_protocol::types::DaemonStore::build_derivation( + guard.client(), + drv, + basic_drv, + harmonia_protocol::daemon_wire::types2::BuildMode::Normal, + )); + while let Some(msg) = result_log.as_mut().next().await { + match msg { + harmonia_protocol::log::LogMessage::Message(m) => { + let mut line = Vec::from(m.text); + line.push(b'\n'); + let _ = log_tx.send(line.into()); + } + harmonia_protocol::log::LogMessage::Result(r) + if matches!( + r.result_type, + harmonia_protocol::log::ResultType::BuildLogLine + | harmonia_protocol::log::ResultType::PostBuildLogLine + ) => + { + for field in &r.fields { + if let harmonia_protocol::log::Field::String(bytes) = field { + let mut line = Vec::from(bytes.as_ref()); + line.push(b'\n'); + let _ = log_tx.send(line.into()); + } + } + } + _ => {} + } + } + drop(log_tx); + result_log.await.wrap_err("build_derivation failed")? + }; + drop(guard); + + // Wait for the log stream to finish. The build_log RPC + // streams to the queue-runner and only resolves once the + // channel is closed (i.e. the build finished). A transport + // error here (e.g. nginx sending an HTTP/2 GOAWAY after + // hitting keepalive_requests) says nothing about whether the + // derivation built, so it is non-fatal. + if let Err(e) = log_handle.await { + tracing::warn!("build log shipping failed for {drv}: {e}"); + } + + // Check for build failure. + use harmonia_protocol::daemon_wire::types2::BuildResultInner; + Ok(match build_result.inner { + BuildResultInner::Success(s) => s, + BuildResultInner::Failure(f) => { + return Err(eyre::eyre!( + "build failed: {:?}: {}", + f.status, + str::from_utf8(&f.error_msg).unwrap_or("Invalid UTF-8") + )); + } + }) + } + #[tracing::instrument(skip(self, m), fields(drv=?m.drv), err)] #[allow(clippy::too_many_lines)] async fn process_build( @@ -396,14 +506,10 @@ impl State { m: BuildMessage, timings: &mut BuildTimings, ) -> Result<(), JobFailure> { - use tokio_stream::StreamExt; - - let store = nix_utils::LocalStore::init(); - let machine_id = self.id; let drv = m .drv - .ok_or(JobFailure::Preparing(anyhow::anyhow!("missing drv")))? + .ok_or(JobFailure::Preparing(eyre::eyre!("missing drv")))? .0; let before_import = Instant::now(); @@ -420,10 +526,22 @@ impl State { step_status: StepStatus::SeningInputs as i32, }) .await; + + // Decode the force-resolved BasicDerivation sent by the queue runner. + let basic_drv: harmonia_store_derivation::derivation::BasicDerivation = m + .resolved_drv + .ok_or(JobFailure::Preparing(eyre::eyre!( + "missing resolved_drv in BuildMessage", + )))? + .try_into() + .map_err(|e: String| { + JobFailure::Import(eyre::eyre!("failed to decode resolved derivation: {e}")) + })?; + + // Fetch the transitive closure of the resolved inputs. let requisites = client - .fetch_drv_requisites(FetchRequisitesRequest { - path: Some(ProtoStorePath::from(drv.clone())), - include_outputs: false, + .fetch_requisites(hydra_proto::StorePaths { + paths: basic_drv.inputs.iter().map(ProtoStorePath::from).collect(), }) .await .map_err(|e| JobFailure::Import(e.into()))? @@ -432,7 +550,7 @@ impl State { import_requisites( &mut client, - store.clone(), + self.pool.clone(), self.metrics.clone(), &gcroot, &drv, @@ -452,99 +570,54 @@ impl State { }) .await; let before_build = Instant::now(); - let (mut child, stdout, stderr) = nix_utils::realise_drv( - &store, - &drv, - &nix_utils::BuildOptions::complete(m.max_log_size, m.max_silent_time, m.build_timeout), - true, - ) - .await - .map_err(|e| JobFailure::Build(e.into()))?; - // The build_log RPC streams stderr to the queue-runner and only - // resolves once the child closes stderr (i.e. the build finished). - // A transport error here (e.g. nginx sending an HTTP/2 GOAWAY after - // hitting keepalive_requests) says nothing about whether the - // derivation builds, so it must not be reported as a BuildFailure - // (which is non-retryable and cascades to every dependent build). - // Map it to Upload so the queue-runner retries the step instead. - // - // TODO: don't restart the build on a log-stream transport error. - // The child is `kill_on_drop`, so returning here aborts and re-runs - // the whole build. Instead, buffer stderr locally, reconnect the - // build_log stream (the queue-runner appends to the same log file - // keyed by drv path), and keep the child running. - client - .build_log(Request::new(crate::utils::compressed_log_stream( - &drv, stderr, - ))) - .await - .map_err(|e| { - JobFailure::Upload( - anyhow::Error::from(e).context("failed to stream build log to queue-runner"), - ) - })?; - - nix_utils::validate_statuscode( - child - .wait() - .await - .map_err(|e| JobFailure::Build(e.into()))?, - ) - .map_err(|e| JobFailure::Build(e.into()))?; - // The process has already finished by this point, so if it takes more than 100ms - // then there probably was no line. - // No need for a loop, since `nix build` only ever prints one line. - let outputs_line = std::pin::pin!(stdout.timeout(tokio::time::Duration::from_millis(100))) - .next() + let success = self + .request_build( + &self.pool, + &drv, + &basic_drv, + m.max_log_size, + m.max_silent_time, + m.build_timeout, + ) .await - .ok_or_else(|| { - JobFailure::PostProcessing(anyhow::anyhow!("Child did not print outputs")) - })? - .map_err(|e| JobFailure::PostProcessing(e.into()))? - .map_err(|e| JobFailure::PostProcessing(e.into()))?; - - let mut output_raw: Vec = serde_json::from_str(&outputs_line) - .map_err(|e| JobFailure::PostProcessing(e.into()))?; - - if output_raw.len() != 1 { - return Err(JobFailure::PostProcessing(anyhow::anyhow!( - "nix built {} derivations, expecting 1", - output_raw.len() - ))); - } + .map_err(JobFailure::Build)?; - let actual_out_drv: nix_utils::StorePath = store - .store_dir() - .parse(&output_raw[0].drv_path) - .map_err(|e: nix_utils::ParseStorePathError| JobFailure::PostProcessing(e.into()))?; - if actual_out_drv != drv { - return Err(JobFailure::PostProcessing(anyhow::anyhow!( - "Nix returned outputs for {actual_out_drv} when we expected {drv}" - ))); - } - - let outputs = output_raw - .pop() - .unwrap() - .outputs + // Extract output paths from the build result. + let outputs: BTreeMap = success + .built_outputs .into_iter() - .map(|(name, path)| { - Ok(( - name, - store.store_dir().parse::(&path)?, - )) - }) - .collect::>>() - .map_err(JobFailure::PostProcessing)?; + .map(|(name, realisation)| (name, realisation.out_path)) + .collect(); for o in outputs.values() { - nix_utils::add_root(&store, &gcroot.root, o); + add_gc_root(&gcroot.root, self.pool.store_dir(), o); } timings.build_elapsed = before_build.elapsed(); tracing::info!("Finished building {drv}"); + // Query path info for each output up front — these are needed both + // for building the result message and are expected to exist for a + // successful build. + let mut output_infos = BTreeMap::new(); + for (name, path) in &outputs { + let info = daemon_client_utils::query_path_info(&self.pool, path) + .await + .wrap_err("query_path_info failed") + .map_err(JobFailure::PostProcessing)? + .ok_or_else(|| { + JobFailure::PostProcessing(eyre::eyre!("missing path info for output `{name}`")) + })?; + output_infos.insert( + name.clone(), + harmonia_store_path_info::ValidPathInfo { + path: path.clone(), + info, + }, + ); + } + let _ = client // we ignore the error here, as this step status has no prio .build_step_update(StepUpdate { build_id: m.build_id.clone(), @@ -555,7 +628,7 @@ impl State { let before_upload = Instant::now(); self.upload_nars( - store.clone(), + self.pool.clone(), outputs.values().cloned().collect::>(), &m.build_id, &machine_id.to_string(), @@ -573,10 +646,10 @@ impl State { }) .await; let build_results = Box::pin(new_success_build_result_info( - store.clone(), + self.pool.clone(), machine_id, &drv, - &outputs, + &output_infos, *timings, m.build_id.clone(), )) @@ -641,20 +714,20 @@ impl State { self.halt.store(true, Ordering::SeqCst); } - #[tracing::instrument(skip(self, store, nars), err)] + #[tracing::instrument(skip(self, pool, nars), err)] async fn upload_nars( &self, - store: nix_utils::LocalStore, - nars: Vec, + pool: harmonia_store_remote::ConnectionPool, + nars: Vec, build_id: &str, machine_id: &str, - presigned_url_opts: Option, - ) -> anyhow::Result<()> { + presigned_url_opts: Option, + ) -> eyre::Result<()> { if let Some(opts) = presigned_url_opts { upload_nars_presigned( self.client.clone(), self.upload_client.clone(), - store, + pool, &nars, opts, build_id, @@ -662,81 +735,88 @@ impl State { ) .await } else { - upload_nars_regular(self.client.clone(), store, self.metrics.clone(), nars).await + upload_nars_regular(self.client.clone(), pool, self.metrics.clone(), nars).await } } } -#[tracing::instrument(skip(store), fields(%gcroot, %path))] -async fn filter_missing( - store: &nix_utils::LocalStore, +#[tracing::instrument(skip(pool), fields(%gcroot, %path))] +async fn is_path_missing( + pool: &harmonia_store_remote::ConnectionPool, gcroot: &Gcroot, - path: nix_utils::StorePath, -) -> Option { - if store.is_valid_path(&path).await { - nix_utils::add_root(store, &gcroot.root, &path); - None + path: StorePath, +) -> eyre::Result> { + if daemon_client_utils::is_valid_path(pool, &path).await? { + add_gc_root(&gcroot.root, pool.store_dir(), &path); + Ok(None) } else { - Some(path) + Ok(Some(path)) } } +/// Keep only paths not yet present in the local store. +async fn filter_missing( + pool: &harmonia_store_remote::ConnectionPool, + gcroot: &Gcroot, + paths: Vec, + concurrency: usize, +) -> eyre::Result> { + use futures::StreamExt as _; + futures::StreamExt::map(tokio_stream::iter(paths), |p| { + is_path_missing(pool, gcroot, p) + }) + .buffered(concurrency) + .collect::>() + .await + .into_iter() + .collect::>>() + .map(|v| v.into_iter().flatten().collect()) +} + +/// Create a GC root symlink for a store path. +/// +/// The symlink target uses the logical store dir, which may dangle +/// outside a chroot but is correct inside it. +fn add_gc_root( + gcroot_dir: &std::path::Path, + store_dir: &harmonia_store_path::StoreDir, + path: &StorePath, +) { + let link = gcroot_dir.join(path.to_string()); + let target = store_dir.display(path).to_string(); + let _ = fs_err::os::unix::fs::symlink(target, &link); +} + async fn substitute_paths( - store: &nix_utils::LocalStore, - paths: &[nix_utils::StorePath], -) -> anyhow::Result<()> { + pool: &harmonia_store_remote::ConnectionPool, + paths: &[StorePath], +) -> eyre::Result<()> { for p in paths { - store.ensure_path(p).await?; + daemon_client_utils::ensure_path(pool, p).await?; } Ok(()) } -fn decode_stream( - stream: S, -) -> impl tokio_stream::Stream> -where - S: tokio_stream::Stream> + Unpin, -{ - let reader = tokio_util::io::StreamReader::new(stream); - let decoder = crate::utils::CompressionDecoder::new(reader); - tokio_util::io::ReaderStream::new(decoder) -} - -#[tracing::instrument(skip(client, store, metrics), fields(%gcroot), err)] +#[tracing::instrument(skip(client, pool, metrics), fields(%gcroot), err)] async fn import_paths( mut client: BuilderClient, - store: nix_utils::LocalStore, + pool: harmonia_store_remote::ConnectionPool, metrics: Arc, gcroot: &Gcroot, - paths: Vec, + paths: Vec, filter: bool, use_substitutes: bool, -) -> anyhow::Result<()> { - use futures::StreamExt as _; - +) -> eyre::Result<()> { let paths = if filter { - futures::StreamExt::map(tokio_stream::iter(paths), |p| { - filter_missing(&store, gcroot, p) - }) - .buffered(10) - .filter_map(|o| async { o }) - .collect::>() - .await + filter_missing(&pool, gcroot, paths, 10).await? } else { paths }; let paths = if use_substitutes { - // we can ignore the error metrics.add_substituting_path(paths.len() as u64); - let _ = substitute_paths(&store, &paths).await; + let _ = substitute_paths(&pool, &paths).await; metrics.sub_substituting_path(paths.len() as u64); - let paths = futures::StreamExt::map(tokio_stream::iter(paths), |p| { - filter_missing(&store, gcroot, p) - }) - .buffered(10) - .filter_map(|o| async { o }) - .collect::>() - .await; + let paths = filter_missing(&pool, gcroot, paths, 10).await?; if paths.is_empty() { return Ok(()); } @@ -745,9 +825,16 @@ async fn import_paths( paths }; - tracing::debug!("Start importing paths"); + if paths.is_empty() { + return Ok(()); + } + + let num_paths = paths.len() as u64; + tracing::debug!("Start importing {num_paths} paths"); + metrics.add_downloading_path(num_paths); + let stream = client - .stream_files(StorePaths { + .fetch_paths(hydra_proto::StorePaths { paths: paths .iter() .map(|p| ProtoStorePath::from(p.clone())) @@ -756,53 +843,43 @@ async fn import_paths( .await? .into_inner(); - metrics.add_downloading_path(paths.len() as u64); - - let byte_stream = tokio_stream::StreamExt::map(stream, |s| { - s.map(|v| bytes::Bytes::from(v.chunk)) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::UnexpectedEof, e)) - }); - let import_result = store.import_paths(decode_stream(byte_stream), false).await; - metrics.sub_downloading_path(paths.len() as u64); - import_result?; - tracing::debug!("Finished importing paths"); + let mut guard = pool.acquire().await?; + let imported = store_transfer::import::import(&mut guard, stream).await?; - for p in paths { - nix_utils::add_root(&store, &gcroot.root, &p); + // Create GC roots while still holding the connection — the + // imported paths are temp-rooted on this connection and can't + // be GC'd until we release it. + for p in &imported { + add_gc_root(&gcroot.root, pool.store_dir(), p); } + drop(guard); + + metrics.sub_downloading_path(num_paths); + tracing::debug!("Finished importing {} paths", imported.len()); Ok(()) } -#[tracing::instrument(skip(client, store, metrics, requisites), fields(%gcroot, %drv), err)] +#[tracing::instrument(skip(client, pool, metrics, requisites), fields(%gcroot, %drv), err)] #[allow(clippy::too_many_arguments)] -async fn import_requisites>( +async fn import_requisites>( client: &mut BuilderClient, - store: nix_utils::LocalStore, + pool: harmonia_store_remote::ConnectionPool, metrics: Arc, gcroot: &Gcroot, - drv: &nix_utils::StorePath, + drv: &StorePath, requisites: T, max_concurrent_downloads: usize, use_substitutes: bool, -) -> anyhow::Result<()> { - use futures::stream::StreamExt as _; +) -> eyre::Result<()> { + let requisites = filter_missing(&pool, gcroot, requisites.into_iter().collect(), 50).await?; - let requisites = futures::StreamExt::map(tokio_stream::iter(requisites), |p| { - filter_missing(&store, gcroot, p) - }) - .buffered(50) - .filter_map(|o| async { o }) - .collect::>() - .await; - - let (input_drvs, input_srcs): (Vec<_>, Vec<_>) = requisites - .into_iter() - .partition(nix_utils::StorePath::is_derivation); + let (input_drvs, input_srcs): (Vec<_>, Vec<_>) = + requisites.into_iter().partition(StorePath::is_derivation); for srcs in input_srcs.chunks(max_concurrent_downloads) { import_paths( client.clone(), - store.clone(), + pool.clone(), metrics.clone(), gcroot, srcs.to_vec(), @@ -815,7 +892,7 @@ async fn import_requisites>( for drvs in input_drvs.chunks(max_concurrent_downloads) { import_paths( client.clone(), - store.clone(), + pool.clone(), metrics.clone(), gcroot, drvs.to_vec(), @@ -825,134 +902,81 @@ async fn import_requisites>( .await?; } - let full_requisites = client - .clone() - .fetch_drv_requisites(FetchRequisitesRequest { - path: Some(ProtoStorePath::from(drv.clone())), - include_outputs: true, - }) - .await? - .into_inner() - .requisites - .into_iter() - .map(|s| s.0) - .collect::>(); - let full_requisites = futures::StreamExt::map(tokio_stream::iter(full_requisites), |p| { - filter_missing(&store, gcroot, p) - }) - .buffered(50) - .filter_map(|o| async { o }) - .collect::>() - .await; - - for other in full_requisites.chunks(max_concurrent_downloads) { - // we can skip filtering here as we already done that - import_paths( - client.clone(), - store.clone(), - metrics.clone(), - gcroot, - other.to_vec(), - false, - use_substitutes, - ) - .await?; - } - Ok(()) } -#[tracing::instrument(skip(client, store, metrics), err)] +#[tracing::instrument(skip(client, pool, metrics), err)] async fn upload_nars_regular( mut client: BuilderClient, - store: nix_utils::LocalStore, + pool: harmonia_store_remote::ConnectionPool, metrics: Arc, - nars: Vec, -) -> anyhow::Result<()> { - // Compute the full closure of output paths so that all referenced - // store paths (e.g. dynamically-created derivations from recursive-nix) - // are uploaded alongside the direct outputs. - let nars = store - .query_requisites(&nars.iter().collect::>(), true) + nars: Vec, +) -> eyre::Result<()> { + // Compute full closure by walking references via daemon protocol. + // query_closure returns ValidPathInfos in dependency order with + // path infos already populated, so we don't need to re-query. + let closure = daemon_client_utils::query_closure(&pool, &nars) .await - .unwrap_or(nars); + .map_err(|e| eyre::eyre!("failed to compute closure: {e}"))?; - let nars = { + // Filter out paths the queue-runner already has. + let closure = { use futures::stream::StreamExt as _; - futures::StreamExt::map(tokio_stream::iter(nars), |p| { + futures::StreamExt::map(tokio_stream::iter(closure), |vpi| { let mut client = client.clone(); async move { if client - .has_path(ProtoStorePath::from(p.clone())) + .has_path(ProtoStorePath::from(vpi.path.clone())) .await .is_ok_and(|r| r.into_inner().has_path) { None } else { - Some(p) + Some(vpi) } } }) .buffered(10) .filter_map(|o| async { o }) - .collect::>() + .collect::>() .await }; - if nars.is_empty() { + if closure.is_empty() { return Ok(()); } tracing::info!("Start uploading paths to queue runner directly"); - let (raw_writer, raw_reader) = tokio::io::duplex(crate::utils::DUPLEX_BUFFER_SIZE); - let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::< + Result, + >(); let before_upload = Instant::now(); - let nars_len = nars.len() as u64; + let nars_len = closure.len() as u64; metrics.add_uploading_path(nars_len); - tokio::task::spawn(async move { - use tokio_stream::StreamExt as _; - let encoder = crate::utils::CompressionEncoder::new(tokio::io::BufReader::new(raw_reader)); - let mut encoded_stream = tokio_util::io::ReaderStream::new(encoder); - while let Some(chunk) = encoded_stream.next().await { - match chunk { - Ok(bytes) => { - if tx - .send(NarData { - chunk: bytes.into(), - }) - .is_err() - { - break; - } - } - Err(e) => { - tracing::error!("Failed to compress NAR chunk: {e}"); - break; - } - } - } - }); + let nars: Vec<_> = closure.iter().map(|vpi| vpi.path.clone()).collect(); + let infos: HashMap<_, _> = closure + .into_iter() + .map(|vpi| (vpi.path, vpi.info)) + .collect(); - let a = client - .build_result(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)) - .map_err(Into::::into); + let export_pool = pool.clone(); + let sender = tokio::spawn(async move { + let mut guard = export_pool.acquire().await?; + store_transfer::export::export(&mut guard, &nars, &infos, &tx).await + }); - let b = tokio::task::spawn_blocking(move || { - let mut sync_writer = tokio_util::io::SyncIoBridge::new(raw_writer); - let closure = move |data: &[u8]| { - use std::io::Write as _; - sync_writer.write_all(data).is_ok() - }; + let upload = client + .build_result(tokio_stream::StreamExt::filter_map( + tokio_stream::wrappers::UnboundedReceiverStream::new(rx), + Result::ok, + )) + .map_err(Into::::into); - store.export_paths(&nars, closure)?; - tracing::debug!("Finished exporting paths"); - Ok::<(), anyhow::Error>(()) - }); - let (a, b) = futures::future::join(a, b).await; - a?; - b??; + let (upload_result, sender_result) = futures::future::join(upload, sender).await; + upload_result?; + sender_result??; tracing::info!( "Finished uploading paths to queue runner directly. elapsed={:?}", @@ -963,43 +987,53 @@ async fn upload_nars_regular( Ok(()) } -#[tracing::instrument(skip(client, store), err)] +#[tracing::instrument(skip(client, pool), err)] async fn upload_nars_presigned( mut client: BuilderClient, upload_client: PresignedUploadClient, - store: nix_utils::LocalStore, - output_paths: &[nix_utils::StorePath], - opts: runner_v1::PresignedUploadOpts, + pool: harmonia_store_remote::ConnectionPool, + output_paths: &[StorePath], + opts: hydra_proto::PresignedUploadOpts, build_id: &str, machine_id: &str, -) -> anyhow::Result<()> { +) -> eyre::Result<()> { use futures::stream::StreamExt as _; tracing::info!("Start uploading paths using presigned urls"); let before_upload = Instant::now(); - let paths_to_upload = store - .query_requisites(&output_paths.iter().collect::>(), true) - .await - .unwrap_or_default(); - let paths_to_upload_ref = paths_to_upload.iter().collect::>(); - let path_infos = Arc::new(store.query_path_infos(&paths_to_upload_ref).await); + // Compute full closure by walking references. Returns path infos + // in dependency order, so no need to re-query. + let closure = daemon_client_utils::query_closure(&pool, output_paths).await?; + + let path_info_map: HashMap<_, _> = closure + .iter() + .map(|vpi| (vpi.path.clone(), vpi.info.clone())) + .collect(); + let paths_to_upload: Vec<_> = closure.iter().map(|vpi| vpi.path.clone()).collect(); + let path_infos = Arc::new(path_info_map); + + let nix_config = daemon_client_utils::parse_nix_remote().ok(); + let debug_store_dir: std::path::PathBuf = nix_config + .as_ref() + .and_then(daemon_client_utils::NixDaemonStoreConfig::real_store_dir) + .unwrap_or_else(|| pool.store_dir().to_string().into()); let mut nars = Vec::with_capacity(paths_to_upload.len()); let mut stream = tokio_stream::iter(paths_to_upload.clone()) .map(|path| { - let store = store.clone(); let path_infos = path_infos.clone(); + let debug_store_dir = debug_store_dir.clone(); async move { let debug_info_ids = if opts.upload_debug_info { - binary_cache::get_debug_info_build_ids(&store, &path).await? + binary_cache::get_debug_info_build_ids(&debug_store_dir, &path).await? } else { Vec::new() }; - let Some(narhash) = path_infos.get(&path).map(|i| i.nar_hash.clone()) else { + let Some(narhash) = path_infos.get(&path).map(|i| i.nar_hash) else { return Ok(None); }; - Ok::<_, anyhow::Error>(Some((path, narhash, debug_info_ids))) + Ok::<_, eyre::Report>(Some((path, narhash, debug_info_ids))) } }) .buffered(10); @@ -1011,7 +1045,7 @@ async fn upload_nars_presigned( } if nars.len() != paths_to_upload.len() { - return Err(anyhow::anyhow!( + return Err(eyre::eyre!( "Mismatch between paths_to_upload ({}) and paths_with_narhash ({})", paths_to_upload.len(), nars.len(), @@ -1023,7 +1057,7 @@ async fn upload_nars_presigned( .await?; if presigned_responses.len() != paths_to_upload.len() { - return Err(anyhow::anyhow!( + return Err(eyre::eyre!( "Mismatch between requested NARs ({}) and presigned URLs ({})", paths_to_upload.len(), presigned_responses.len() @@ -1032,8 +1066,9 @@ async fn upload_nars_presigned( for presigned_response in presigned_responses { upload_single_nar_presigned( - &store, - &nix_utils::parse_store_path(&presigned_response.store_path), + &pool, + &StorePath::from_base_path(&presigned_response.store_path) + .map_err(|e| eyre::eyre!("invalid store path in presigned response: {e}"))?, build_id, machine_id, &presigned_response, @@ -1050,21 +1085,27 @@ async fn upload_nars_presigned( Ok(()) } -#[tracing::instrument(skip(store, nar_path, presigned_response), err)] +#[tracing::instrument(skip(pool, nar_path, presigned_response), err)] async fn upload_single_nar_presigned( - store: &nix_utils::LocalStore, - nar_path: &nix_utils::StorePath, + pool: &harmonia_store_remote::ConnectionPool, + nar_path: &StorePath, build_id: &str, machine_id: &str, - presigned_response: &runner_v1::PresignedNarResponse, + presigned_response: &hydra_proto::PresignedNarResponse, client: &mut BuilderClient, upload_client: &PresignedUploadClient, -) -> anyhow::Result<()> { - let narinfo = binary_cache::path_to_narinfo(store, nar_path).await?; +) -> eyre::Result<()> { + // Presigned upload requires constructing NarInfo from daemon path info. + let narinfo: binary_cache::NarInfo = { + let info = daemon_client_utils::query_path_info(pool, nar_path) + .await? + .ok_or_else(|| eyre::eyre!("path not found: {nar_path}"))?; + binary_cache::narinfo_simple(nar_path, info, Compression::None) + }; let nar_upload = presigned_response .nar_upload .as_ref() - .ok_or_else(|| anyhow::anyhow!("nar_upload information is missing"))?; + .ok_or_else(|| eyre::eyre!("nar_upload information is missing"))?; let presigned_request = binary_cache::PresignedUploadResponse { nar_url: presigned_response.nar_url.clone(), @@ -1097,36 +1138,21 @@ async fn upload_single_nar_presigned( }; let updated_narinfo = upload_client - .process_presigned_request(store, narinfo, presigned_request) + .process_presigned_request(pool, narinfo, presigned_request) .await?; tracing::debug!( "Successfully uploaded presigned NAR for {} to {}", nar_path, - updated_narinfo.url + updated_narinfo.info.url.as_deref().unwrap_or("") ); - if let (Some(file_hash), Some(file_size)) = ( - updated_narinfo.file_hash.as_ref(), - updated_narinfo.file_size, - ) { - let completion_msg = runner_v1::PresignedUploadComplete { + if updated_narinfo.info.download_hash.is_some() && updated_narinfo.info.download_size.is_some() + { + let completion_msg = hydra_proto::PresignedUploadComplete { build_id: build_id.to_owned(), machine_id: machine_id.to_owned(), - store_path: nar_path.to_string().clone(), - url: updated_narinfo.url.clone(), - compression: updated_narinfo.compression.as_str().to_owned(), - file_hash: format!("{}", file_hash.as_base32()), - file_size, - nar_hash: format!("{}", updated_narinfo.nar_hash.as_base32()), - nar_size: updated_narinfo.nar_size, - references: updated_narinfo - .references - .iter() - .map(|p| p.to_string().clone()) - .collect(), - deriver: updated_narinfo.deriver.map(|p| p.to_string().clone()), - ca: updated_narinfo.ca, + nar_info: Some((&updated_narinfo).into()), }; client @@ -1137,36 +1163,54 @@ async fn upload_single_nar_presigned( Ok(()) } -#[tracing::instrument(skip(store, outputs), fields(%drv), ret(level = tracing::Level::DEBUG), err)] +#[tracing::instrument(skip(pool, output_infos), fields(%drv), ret(level = tracing::Level::DEBUG), err)] async fn new_success_build_result_info( - store: nix_utils::LocalStore, + pool: harmonia_store_remote::ConnectionPool, machine_id: uuid::Uuid, - drv: &nix_utils::StorePath, - outputs: &BTreeMap, + drv: &StorePath, + output_infos: &BTreeMap, timings: BuildTimings, build_id: String, -) -> anyhow::Result { - let pathinfos = store - .query_path_infos(&outputs.values().collect::>()) - .await; - let nix_support = Box::pin(shared::parse_nix_support_from_outputs(&store, outputs)).await?; - - let mut build_outputs = vec![]; - for (name, path) in outputs { - build_outputs.push(Output { - output: Some(match pathinfos.get(path) { - Some(info) => output::Output::Withpath(OutputWithPath { - name: name.to_string(), - closure_size: store.compute_closure_size(path).await, - path: Some(ProtoStorePath::from(path.clone())), - nar_size: info.nar_size, - nar_hash: info.nar_hash.clone(), - }), - None => output::Output::Nameonly(OutputNameOnly { - name: name.to_string(), - }), - }), - }); +) -> eyre::Result { + let outputs: BTreeMap<_, _> = output_infos + .iter() + .map(|(name, vpi)| (name.clone(), vpi.path.clone())) + .collect(); + let real_store_dir = daemon_client_utils::parse_nix_remote() + .ok() + .and_then(|c| c.real_store_dir()) + .unwrap_or_else(|| pool.store_dir().to_string().into()); + let real_store_path = &real_store_dir; + let fs = nix_support::FilesystemOperations { + real_store_dir: real_store_path.to_owned(), + }; + let per_output_nix_support = Box::pin(nix_support::parse_nix_support_from_outputs( + pool.store_dir(), + real_store_path, + &fs, + &outputs, + )) + .await?; + + let mut result_infos = std::collections::HashMap::new(); + for (name, vpi) in output_infos { + let ns = per_output_nix_support + .get(name) + .cloned() + .unwrap_or_default(); + result_infos.insert( + name.to_string(), + OutputInfo { + path: Some(ProtoStorePath::from(vpi.path.clone())), + closure_size: daemon_client_utils::compute_closure_size(&pool, &vpi.path).await, + nar_size: vpi.info.nar_size, + nar_hash: { + let h: harmonia_utils_hash::Hash = vpi.info.nar_hash.into(); + Some((&h).into()) + }, + nix_support: Some(ns.into()), + }, + ); } Ok(BuildResultInfo { @@ -1176,34 +1220,6 @@ async fn new_success_build_result_info( build_time_ms: u64::try_from(timings.build_elapsed.as_millis())?, upload_time_ms: u64::try_from(timings.upload_elapsed.as_millis())?, result_state: BuildResultState::Success as i32, - outputs: build_outputs, - nix_support: Some(NixSupport { - metrics: nix_support - .metrics - .into_iter() - .map(|m| BuildMetric { - path: m.path, - name: m.name, - unit: m.unit, - value: m.value, - }) - .collect(), - failed: nix_support.failed, - hydra_release_name: nix_support.hydra_release_name, - products: nix_support - .products - .into_iter() - .map(|p| BuildProduct { - path: p.path, - default_path: p.default_path, - r#type: p.r#type, - subtype: p.subtype, - name: p.name, - is_regular: p.is_regular, - sha256hash: p.sha256hash, - file_size: p.file_size, - }) - .collect(), - }), + output_infos: result_infos, }) } diff --git a/subprojects/hydra-builder/src/system.rs b/subprojects/hydra-builder/src/system.rs index 8225c7695..7e09981fe 100644 --- a/subprojects/hydra-builder/src/system.rs +++ b/subprojects/hydra-builder/src/system.rs @@ -1,3 +1,4 @@ +use color_eyre::eyre; use hashbrown::HashMap; use procfs_core::FromRead as _; @@ -11,7 +12,7 @@ pub struct BaseSystemInfo { impl BaseSystemInfo { #[cfg(target_os = "linux")] #[tracing::instrument(err)] - pub fn new() -> anyhow::Result { + pub fn new() -> eyre::Result { let cpuinfo = procfs_core::CpuInfo::from_file("/proc/cpuinfo")?; let meminfo = procfs_core::Meminfo::from_file("/proc/meminfo")?; let bogomips = cpuinfo @@ -29,7 +30,7 @@ impl BaseSystemInfo { #[cfg(target_os = "macos")] #[tracing::instrument(err)] - pub fn new() -> anyhow::Result { + pub fn new() -> eyre::Result { let mut sys = sysinfo::System::new_all(); sys.refresh_memory(); sys.refresh_cpu_all(); @@ -42,47 +43,18 @@ impl BaseSystemInfo { } } -#[derive(Debug, Clone, Copy)] -pub struct Pressure { - pub avg10: f32, - pub avg60: f32, - pub avg300: f32, - pub total: u64, -} +pub use hydra_proto::{Pressure, PressureState}; #[cfg(target_os = "linux")] -impl Pressure { - const fn new(record: &procfs_core::PressureRecord) -> Self { - Self { - avg10: record.avg10, - avg60: record.avg60, - avg300: record.avg300, - total: record.total, - } +fn pressure_from_record(record: &procfs_core::PressureRecord) -> Pressure { + Pressure { + avg10: record.avg10, + avg60: record.avg60, + avg300: record.avg300, + total: record.total, } } -impl From for crate::grpc::runner_v1::Pressure { - fn from(val: Pressure) -> Self { - Self { - avg10: val.avg10, - avg60: val.avg60, - avg300: val.avg300, - total: val.total, - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct PressureState { - pub cpu_some: Option, - pub mem_some: Option, - pub mem_full: Option, - pub io_some: Option, - pub io_full: Option, - pub irq_full: Option, -} - // TODO: remove once https://github.com/eminence/procfs/issues/351 is resolved // Next 3 Functions are copied from https://github.com/eminence/procfs/blob/v0.17.0/procfs-core/src/pressure.rs#L93 // LICENSE is Apache2.0/MIT @@ -135,29 +107,27 @@ fn parse_pressure_record(line: &str) -> procfs_core::ProcResult Option { - if !fs_err::exists("/proc/pressure").unwrap_or_default() { - return None; - } - - let cpu_psi = procfs_core::CpuPressure::from_file("proc/pressure/cpu").ok(); - let mem_psi = procfs_core::MemoryPressure::from_file("/proc/pressure/memory").ok(); - let io_psi = procfs_core::IoPressure::from_file("/proc/pressure/io").ok(); - let irq_psi_full = fs_err::read_to_string("/proc/pressure/irq") - .ok() - .and_then(|v| parse_pressure_record(&v).ok()); - - Some(Self { - cpu_some: cpu_psi.map(|v| Pressure::new(&v.some)), - mem_some: mem_psi.as_ref().map(|v| Pressure::new(&v.some)), - mem_full: mem_psi.map(|v| Pressure::new(&v.full)), - io_some: io_psi.as_ref().map(|v| Pressure::new(&v.some)), - io_full: io_psi.map(|v| Pressure::new(&v.full)), - irq_full: irq_psi_full.map(|v| Pressure::new(&v)), - }) +#[must_use] +pub fn read_pressure_state() -> Option { + if !fs_err::exists("/proc/pressure").unwrap_or_default() { + return None; } + + let cpu_psi = procfs_core::CpuPressure::from_file("proc/pressure/cpu").ok(); + let mem_psi = procfs_core::MemoryPressure::from_file("/proc/pressure/memory").ok(); + let io_psi = procfs_core::IoPressure::from_file("/proc/pressure/io").ok(); + let irq_psi_full = fs_err::read_to_string("/proc/pressure/irq") + .ok() + .and_then(|v| parse_pressure_record(&v).ok()); + + Some(PressureState { + cpu_some: cpu_psi.map(|v| pressure_from_record(&v.some)), + mem_some: mem_psi.as_ref().map(|v| pressure_from_record(&v.some)), + mem_full: mem_psi.map(|v| pressure_from_record(&v.full)), + io_some: io_psi.as_ref().map(|v| pressure_from_record(&v.some)), + io_full: io_psi.map(|v| pressure_from_record(&v.full)), + irq_full: irq_psi_full.as_ref().map(pressure_from_record), + }) } #[derive(Debug, Clone, Copy)] @@ -174,7 +144,7 @@ pub struct SystemLoad { } #[tracing::instrument(err)] -pub fn get_mount_free_percent(dest: &str) -> anyhow::Result { +pub fn get_mount_free_percent(dest: &str) -> eyre::Result { let stat = nix::sys::statvfs::statvfs(dest)?; let total_bytes = (stat.blocks() as u64) * stat.block_size(); @@ -186,38 +156,28 @@ pub fn get_mount_free_percent(dest: &str) -> anyhow::Result { impl SystemLoad { #[cfg(target_os = "linux")] #[tracing::instrument(err)] - pub fn new(build_dir: &str) -> anyhow::Result { + pub fn new(build_dir: &str, store_dir: &str) -> eyre::Result { let meminfo = procfs_core::Meminfo::from_file("/proc/meminfo")?; let load = procfs_core::LoadAverage::from_file("/proc/loadavg")?; - let store_dir = nix_utils::StoreDir::new( - std::env::var("NIX_STORE_DIR").unwrap_or_else(|_| "/nix/store".to_owned()), - ) - .unwrap_or_default(); - Ok(Self { load_avg_1: load.one, load_avg_5: load.five, load_avg_15: load.fifteen, mem_usage: meminfo.mem_total - meminfo.mem_available.unwrap_or(0), - pressure: PressureState::new(), + pressure: read_pressure_state(), build_dir_free_percent: get_mount_free_percent(build_dir).unwrap_or(100.), - store_free_percent: get_mount_free_percent(store_dir.to_str()).unwrap_or(100.), + store_free_percent: get_mount_free_percent(store_dir).unwrap_or(100.), }) } #[cfg(target_os = "macos")] #[tracing::instrument(err)] - pub fn new(build_dir: &str) -> anyhow::Result { + pub fn new(build_dir: &str, store_dir: &str) -> eyre::Result { let mut sys = sysinfo::System::new_all(); sys.refresh_memory(); let load = sysinfo::System::load_average(); - let store_dir = nix_utils::StoreDir::new( - std::env::var("NIX_STORE_DIR").unwrap_or_else(|_| "/nix/store".to_owned()), - ) - .unwrap_or_default(); - Ok(Self { load_avg_1: load.one as f32, load_avg_5: load.five as f32, @@ -225,7 +185,7 @@ impl SystemLoad { mem_usage: sys.used_memory(), pressure: None, build_dir_free_percent: get_mount_free_percent(build_dir).unwrap_or(0.), - store_free_percent: get_mount_free_percent(store_dir.to_str()).unwrap_or(0.), + store_free_percent: get_mount_free_percent(store_dir).unwrap_or(0.), }) } } diff --git a/subprojects/hydra-builder/src/utils.rs b/subprojects/hydra-builder/src/utils.rs index a023f5335..87e50d267 100644 --- a/subprojects/hydra-builder/src/utils.rs +++ b/subprojects/hydra-builder/src/utils.rs @@ -1,17 +1,21 @@ +use harmonia_store_path::StorePath; use tokio::io::BufReader; -use tokio_stream::wrappers::LinesStream; use tokio_util::io::ReaderStream; -use crate::grpc::runner_v1::LogChunk; -use shared::proto::ProtoStorePath; +use hydra_proto::LogChunk; +use hydra_proto::ProtoStorePath; pub(crate) type CompressionEncoder = async_compression::tokio::bufread::ZstdEncoder; -pub(crate) type CompressionDecoder = async_compression::tokio::bufread::ZstdDecoder; -pub(crate) const DUPLEX_BUFFER_SIZE: usize = 256 * 1024; +const DUPLEX_BUFFER_SIZE: usize = 256 * 1024; +/// Build a gRPC `LogChunk` stream from a channel of raw log bytes. +/// +/// Log data received on `rx` is zstd-compressed and yielded as +/// `LogChunk` messages. The first chunk carries the drv path so the +/// server knows which build the log belongs to. pub(crate) fn compressed_log_stream( - drv: &nix_utils::StorePath, - log_output: LinesStream>, + drv: &StorePath, + rx: tokio::sync::mpsc::UnboundedReceiver, ) -> impl tokio_stream::Stream + use<> { let (raw_writer, raw_reader) = tokio::io::duplex(DUPLEX_BUFFER_SIZE); @@ -19,21 +23,12 @@ pub(crate) fn compressed_log_stream( use tokio::io::AsyncWriteExt as _; use tokio_stream::StreamExt as _; - let mut log_output = log_output; + let mut rx = tokio_stream::wrappers::UnboundedReceiverStream::new(rx); let mut raw_writer = raw_writer; - while let Some(chunk) = log_output.next().await { - match chunk { - Ok(line) => { - let data = format!("{line}\n"); - if raw_writer.write_all(data.as_bytes()).await.is_err() { - break; - } - } - Err(e) => { - tracing::error!("Failed to read log chunk: {e}"); - break; - } + while let Some(chunk) = rx.next().await { + if raw_writer.write_all(&chunk).await.is_err() { + break; } } }); @@ -49,7 +44,7 @@ pub(crate) fn compressed_log_stream( match chunk { Ok(bytes) => yield LogChunk { // Only the first chunk needs the drv; server reads it once - drv: if first { first = false; Some(drv.clone()) } else {None}, + drv: if first { first = false; Some(drv.clone()) } else { None }, data: bytes.into(), }, Err(e) => { diff --git a/subprojects/hydra-manual/architecture.md b/subprojects/hydra-manual/architecture.md deleted file mode 100644 index ec67bd37a..000000000 --- a/subprojects/hydra-manual/architecture.md +++ /dev/null @@ -1,129 +0,0 @@ -This is a rough overview from informal discussions and explanations of inner workings of Hydra. -You can use it as a guide to navigate the codebase or ask questions. - -## Architecture - -### Components - -- Postgres database - - configuration - - build queue - - what is already built - - what is going to build -- `hydra-server` - - Perl, Catalyst - - web frontend -- `hydra-evaluator` - - Perl, C++ - - fetches repositories - - evaluates job sets - - pointers to a repository - - adds builds to the queue -- `hydra-queue-runner` - - C++ - - monitors the queue - - executes build steps - - uploads build results - - copy to a Nix store -- Nix store - - contains `.drv`s - - populated by `hydra-evaluator` - - read by `hydra-queue-runner` -- destination Nix store - - can be a binary cache - - e.g. `[cache.nixos.org](http://cache.nixos.org)` or the same store again (for small Hydra instances) -- plugin architecture - - extend evaluator for new kinds of repositories - - e.g. fetch from `git` - -### Database Schema - -[https://github.com/NixOS/hydra/blob/master/src/sql/hydra.sql](https://github.com/NixOS/hydra/blob/master/src/sql/hydra.sql) - -- `Jobsets` - - populated by calling Nix evaluator - - every Nix derivation in `release.nix` is a Job - - `flake` - - URL to flake, if job is from a flake - - single-point of configuration for flake builds - - flake itself contains pointers to dependencies - - for other builds we need more configuration data -- `JobsetInputs` - - more configuration for a Job -- `JobsetInputAlts` - - historical, where you could have more than one alternative for each input - - it would have done the cross product of all possibilities - - not used any more, as now every input is unique - - originally that was to have alternative values for the system parameter - - `x86-linux`, `x86_64-darwin` - - turned out not to be a good idea, as job set names did not uniquely identify output -- `Builds` - - queue: scheduled and finished builds - - instance of a Job - - corresponds to a top-level derivation - - can have many dependencies that don’t have a corresponding build - - dependencies represented as `BuildSteps` - - a Job is all the builds with a particular name, e.g. - - `git.x86_64-linux` is a job - - there maybe be multiple builds for that job - - build ID: just an auto-increment number - - building one thing can actually cause many (hundreds of) derivations to be built - - for queued builds, the `drv` has to be present in the store - - otherwise build will fail, e.g. after garbage collection -- `BuildSteps` - - corresponds to a derivation or substitution - - are reused through the Nix store - - may be duplicated for unique derivations due to how they relate to `Jobs` -- `BuildStepOutputs` - - corresponds directly to derivation outputs - - `out`, `dev`, ... -- `BuildProducts` - - not a Nix concept - - populated from a special file `$out/nix-support/hydra-build-producs` - - used to scrape parts of build results out to the web frontend - - e.g. manuals, ISO images, etc. -- `BuildMetrics` - - scrapes data from magic location, similar to `BuildProducts` to show fancy graphs - - e.g. test coverage, build times, CPU utilization for build - - `$out/nix-support/hydra-metrics` -- `BuildInputs` - - probably obsolute -- `JobsetEvalMembers` - - joins evaluations with jobs - - huge table, 10k’s of entries for one `nixpkgs` evaluation - - can be imagined as a subset of the eval cache - - could in principle use the eval cache - -### `release.nix` - -- hydra-specific convention to describe the build -- should evaluate to an attribute set that contains derivations -- hydra considers every attribute in that set a job -- every job needs a unique name - - if you want to build for multiple platforms, you need to reflect that in the name -- hydra does a deep traversal of the attribute set - - just evaluating the names may take half an hour - -## FAQ - -Can we imagine Hydra to be a persistence layer for the build graph? - -- partially, it lacks a lot of information - - does not keep edges of the build graph - -How does Hydra relate to `nix build`? - -- reimplements the top level Nix build loop, scheduling, etc. -- Hydra has to persist build results -- Hydra has more sophisticated remote build execution and scheduling than Nix - -Is it conceptually possible to unify Hydra’s capabilities with regular Nix? - -- Nix does not have any scheduling, it just traverses the build graph -- Hydra has scheduling in terms of job set priorities, tracks how much of a job set it has worked on - - makes sure jobs don’t starve each other -- Nix cannot dynamically add build jobs at runtime - - [RFC 92](https://github.com/NixOS/rfcs/blob/master/rfcs/0092-plan-dynamism.md) should enable that - - internally it is already possible, but there is no interface to do that -- Hydra queue runner is a long running process - - Nix takes a static set of jobs, working it off at once diff --git a/subprojects/hydra-manual/src/SUMMARY.md b/subprojects/hydra-manual/src/SUMMARY.md index 5ef8ac909..663bf3fcb 100644 --- a/subprojects/hydra-manual/src/SUMMARY.md +++ b/subprojects/hydra-manual/src/SUMMARY.md @@ -14,7 +14,10 @@ - [Monitoring Hydra](./monitoring/README.md) ## Developer's Guide + +- [Architecture](architecture.md) - [Hacking](hacking.md) - [Hydra Notifications](notifications.md) + ----------- [About](about.md) diff --git a/subprojects/hydra-manual/src/api.md b/subprojects/hydra-manual/src/api.md index 1e27c644c..87225e6ff 100644 --- a/subprojects/hydra-manual/src/api.md +++ b/subprojects/hydra-manual/src/api.md @@ -1,18 +1,14 @@ Using the external API ====================== -To be able to create integrations with other services, Hydra exposes an -external API that you can manage projects with. +To be able to create integrations with other services, Hydra exposes an external API that you can manage projects with. -The API is accessed over HTTP(s) where all data is sent and received as -JSON. +The API is accessed over HTTP(s) where all data is sent and received as JSON. -Creating resources requires the caller to be authenticated, while -retrieving resources does not. +Creating resources requires the caller to be authenticated, while retrieving resources does not. -The API does not have a separate URL structure for it\'s endpoints. -Instead you request the pages of the web interface as `application/json` -to use the API. +The API does not have a separate URL structure for its endpoints. +Instead you request the pages of the web interface as `application/json` to use the API. List projects ------------- @@ -22,8 +18,7 @@ To list all the `projects` of the Hydra install: GET / Accept: application/json -This will give you a list of `projects`, where each `project` contains -general information and a list of its `job sets`. +This will give you a list of `projects`, where each `project` contains general information and a list of its `job sets`. **Example** diff --git a/subprojects/hydra-manual/src/architecture.md b/subprojects/hydra-manual/src/architecture.md new file mode 100644 index 000000000..9da091af6 --- /dev/null +++ b/subprojects/hydra-manual/src/architecture.md @@ -0,0 +1,227 @@ +# Architecture + +This is an overview of Hydra's inner workings. +You can use it as a guide to navigate the codebase or ask questions. + +## Components + +Hydra's components are split across a coordinator machine and any number of builder machines. +The NixOS modules in `nixos-modules/` reflect this split: `web-app` and `queue-runner` run on the master, while `builder` runs on remote machines. +For small installations, all three can run on a single host (the `hydra` module combines them). +But most installation will want to use multiple build machines for scale. + +### Coordinator machine + +These components all share a single Nix store and PostgreSQL database on the master: + +- **PostgreSQL database** + - stores configuration, the build queue (scheduled and finished builds), and results +- **`hydra-server`** (Perl, Catalyst) + - web frontend and REST API + - user authentication (built-in or LDAP) +- **`hydra-evaluator`** (C++) + - periodically evaluates jobsets by invoking the Nix evaluator + - writes `.drv` files into the coordinator's Nix store + - adds new builds to the queue when evaluation results change +- **`hydra-eval-jobset`** (Perl) + - called by the evaluator to orchestrate fetching inputs and running the Nix evaluation +- **`hydra-queue-runner`** (Rust) + - reads `.drv` files from the coordinator's Nix store + - schedules build steps across builders + - uploads results to a destination store + - exposes a gRPC service that builders connect to +- **`hydra-notify`** (Perl) + - dispatches post-build notifications to plugins (email, GitHub/GitLab status, Slack, etc.) + - listens for PostgreSQL `NOTIFY` events from the queue runner +- **Plugin system** (Perl) + - input plugins extend the evaluator with new source types (Git, Mercurial, Darcs, etc.) + - notification plugins react to build lifecycle events + +### Destination store + +The queue runner uploads built outputs to a *destination store*, which is separate from the coordinator's local Nix store. +This can be an S3-compatible binary cache, or for small installations it can just be the coordinator's own store. +See [Populating a Cache](configuration.md#populating-a-cache) for configuration. + +### Builder machines + +Builders have their own Nix store — they do not need access to the coordinator's store or database. + +- **`hydra-builder`** (Rust) + - build execution agent that runs on remote machines + - connects to the queue runner's gRPC service + - receives derivations to build, streams back logs and results + +## Rust crate dependencies + +The following is the [transitive reduction](https://en.wikipedia.org/wiki/Transitive_reduction) of the dependency graph between the Rust crates in this repo. +Solid arrows are normal dependencies; dashed arrows are dev (test-only) dependencies. + + + +```mermaid +graph BT + binary-cache --> daemon-client-utils + nix-support --> store-path-utils + db --> nix-support + hydra-proto --> nix-support + store-transfer --> hydra-proto + hydra-builder --> binary-cache + hydra-builder --> hydra-tracing + hydra-builder --> store-transfer + hydra-queue-runner --> binary-cache + hydra-queue-runner --> db + hydra-queue-runner --> hydra-tracing + hydra-queue-runner --> store-transfer + binary-cache -.-> hydra-tracing + db -.-> test-utils +``` + +### Shared Rust libraries + +- `hydra-proto`: + generated gRPC/protobuf code for the builder ↔ queue-runner interface (message types, client stubs, server traits) + +- `db`: + PostgreSQL database access via SQLx (models, queries, connection pooling) + +- `binary-cache`: + reading and writing Nix binary cache artifacts (NARinfo, NAR files, signatures, presigned uploads) + +- `daemon-client-utils`: + Various utilities for working with the daemon connection beyond what the Harmonia libraries provide. + +- `store-transfer`: + shared import/export logic for streaming store objects as `AddToStoreRequest` protobuf messages, used by both the builder and queue-runner + +- `nix-support`: + Infrastructure for interpreting the `${store_object}/nix-support` directory convention + +- `store-path-utils`: + lightweight store path utilities built on harmonia types + +- `tracing`: + OpenTelemetry/tracing setup with optional gRPC export + +- `test-utils`: + test fixtures and helpers for integration tests + +## Source layout + +The repository is organized into subprojects: + +- [`subprojects/hydra/`](https://github.com/NixOS/hydra/tree/master/subprojects/hydra) + — the Perl/Catalyst web application, evaluator scripts, SQL schema, and plugins +- [`subprojects/hydra-queue-runner/`](https://github.com/NixOS/hydra/tree/master/subprojects/hydra-queue-runner) + — the Rust queue runner +- [`subprojects/hydra-builder/`](https://github.com/NixOS/hydra/tree/master/subprojects/hydra-builder) + — the Rust build agent +- [`subprojects/crates/`](https://github.com/NixOS/hydra/tree/master/subprojects/crates) + — shared Rust libraries +- [`subprojects/proto/`](https://github.com/NixOS/hydra/tree/master/subprojects/proto) + — Protocol Buffer `.proto` source files (compiled by the `proto` crate's build script) +- [`subprojects/nix-perl/`](https://github.com/NixOS/hydra/tree/master/subprojects/nix-perl) + — Perl bindings to the Nix store API +- [`subprojects/hydra-tests/`](https://github.com/NixOS/hydra/tree/master/subprojects/hydra-tests) + — integration test suite (Perl, using `Test::More`) +- [`subprojects/hydra-manual/`](https://github.com/NixOS/hydra/tree/master/subprojects/hydra-manual) + — this manual (mdbook) + +The build system uses Meson for the C++ and Perl components and Cargo for the Rust workspace. + +## Database Schema + +The canonical schema lives in [`subprojects/hydra/sql/hydra.sql`](https://github.com/NixOS/hydra/blob/master/subprojects/hydra/sql/hydra.sql). +Incremental migrations are in `migrations/upgrade-N.sql`; see the [SQL README](https://github.com/NixOS/hydra/blob/master/subprojects/hydra/sql/README.md) for details on making schema changes. + +The database is accessed by all three language runtimes: Perl (DBI/DBIx::Class), C++ (libpqxx), and Rust (SQLx). + +Key tables: + +- `Jobsets` + - populated by calling Nix evaluator + - every Nix derivation in `release.nix` is a Job + - `flake` + - URL to flake, if job is from a flake + - single-point of configuration for flake builds + - flake itself contains pointers to dependencies + - for other builds we need more configuration data +- `JobsetInputs` + - more configuration for a Job +- `JobsetInputAlts` + - historical, where you could have more than one alternative for each input + - it would have done the cross product of all possibilities + - not used any more, as now every input is unique + - originally that was to have alternative values for the system parameter + - `x86-linux`, `x86_64-darwin` + - turned out not to be a good idea, as job set names did not uniquely identify output +- `Builds` + - queue: scheduled and finished builds + - instance of a Job + - corresponds to a top-level derivation + - can have many dependencies that don’t have a corresponding build + - dependencies represented as `BuildSteps` + - a Job is all the builds with a particular name, e.g. + - `git.x86_64-linux` is a job + - there maybe be multiple builds for that job + - build ID: just an auto-increment number + - building one thing can actually cause many (hundreds of) derivations to be built + - for queued builds, the `drv` has to be present in the store + - otherwise build will fail, e.g. after garbage collection +- `BuildSteps` + - corresponds to a derivation or substitution + - are reused through the Nix store + - may be duplicated for unique derivations due to how they relate to `Jobs` +- `BuildStepOutputs` + - corresponds directly to derivation outputs + - `out`, `dev`, ... +- `BuildProducts` + - not a Nix concept + - populated from a special file `$out/nix-support/hydra-build-products` + - used to scrape parts of build results out to the web frontend + - e.g. manuals, ISO images, etc. +- `BuildMetrics` + - scrapes data from magic location, similar to `BuildProducts` to show fancy graphs + - e.g. test coverage, build times, CPU utilization for build + - `$out/nix-support/hydra-metrics` +- `BuildInputs` + - probably obsolete +- `JobsetEvalMembers` + - joins evaluations with jobs + - huge table, 10k’s of entries for one `nixpkgs` evaluation + - can be imagined as a subset of the eval cache + - could in principle use the eval cache + +## `release.nix` + +- hydra-specific convention to describe the build +- should evaluate to an attribute set that contains derivations +- hydra considers every attribute in that set a job +- every job needs a unique name + - if you want to build for multiple platforms, you need to reflect that in the name +- hydra does a deep traversal of the attribute set + - just evaluating the names may take half an hour + +## FAQ + +Can we imagine Hydra to be a persistence layer for the build graph? + +- partially, it lacks a lot of information + - does not keep edges of the build graph + +How does Hydra relate to `nix build`? + +- reimplements the top level Nix build loop, scheduling, etc. +- Hydra has to persist build results +- Hydra has more sophisticated remote build execution and scheduling than Nix + +Is it conceptually possible to unify Hydra’s capabilities with regular Nix? + +- Nix does not have any scheduling, it just traverses the build graph +- Hydra has scheduling in terms of job set priorities, tracks how much of a job set it has worked on + - makes sure jobs don’t starve each other +- Both Hydra and Nix can dynamically add build jobs at runtime + - Hydra queued up new jobs dynamically / on-line long before Nix. + - But now, both Nix and Hydra now have experimental support for [dynamic derivations](https://github.com/NixOS/rfcs/blob/master/rfcs/0092-plan-dynamism.md), where build jobs can produce new derivations at build time +- Hydra queue runner is a long running process + - Nix takes a static set of jobs, working it off at once diff --git a/subprojects/hydra-manual/src/configuration.md b/subprojects/hydra-manual/src/configuration.md index 491376b3e..fa2da07fb 100644 --- a/subprojects/hydra-manual/src/configuration.md +++ b/subprojects/hydra-manual/src/configuration.md @@ -1,11 +1,9 @@ Configuration ============= -This chapter is a collection of configuration snippets for different -scenarios. +This chapter is a collection of configuration snippets for different scenarios. -The configuration is parsed by `Config::General` which has [a pretty -thorough documentation on their file format](https://metacpan.org/pod/Config::General#CONFIG-FILE-FORMAT). +The configuration is parsed by `Config::General` which has [a pretty thorough documentation on their file format](https://metacpan.org/pod/Config::General#CONFIG-FILE-FORMAT). Hydra calls the parser with the following options: - `-UseApacheInclude => 1` - `-IncludeAgain => 1` @@ -14,8 +12,8 @@ Hydra calls the parser with the following options: Including files --------------- -`hydra.conf` supports Apache-style includes. This is **IMPORTANT** -because that is how you keep your **secrets** out of the **Nix store**. +`hydra.conf` supports Apache-style includes. +This is **IMPORTANT** because that is how you keep your **secrets** out of the **Nix store**. Hopefully this got your attention 😌 This: @@ -28,9 +26,7 @@ should **NOT** be in `hydra.conf`. `hydra.conf` is rendered in the Nix store and is therefore world-readable. -Instead, the above should be written to a file outside the Nix store by -other means (manually, using Nixops' secrets feature, etc) and included -like so: +Instead, the above should be written to a file outside the Nix store by other means (manually, using Nixops' secrets feature, etc) and included like so: ``` Include /run/keys/hydra/github_authorizations.conf ``` @@ -38,8 +34,7 @@ Include /run/keys/hydra/github_authorizations.conf Serving behind reverse proxy ---------------------------- -To serve hydra web server behind reverse proxy like *nginx* or *httpd* -some additional configuration must be made. +To serve hydra web server behind reverse proxy like *nginx* or *httpd* some additional configuration must be made. Edit your `hydra.conf` file in a similar way to this example: @@ -48,16 +43,13 @@ using_frontend_proxy 1 base_uri example.com ``` -`base_uri` should be your hydra servers proxied URL. If you are using -Hydra nixos module then setting `hydraURL` option should be enough. +`base_uri` should be your hydra servers proxied URL. +If you are using Hydra nixos module then setting `hydraURL` option should be enough. -You also need to configure your reverse proxy to pass `X-Request-Base` -to hydra, with the same value as `base_uri`. -This also covers the case of serving Hydra with a prefix path, -as in [http://example.com/hydra](). +You also need to configure your reverse proxy to pass `X-Request-Base` to hydra, with the same value as `base_uri`. +This also covers the case of serving Hydra with a prefix path, as in [http://example.com/hydra](). -For example if you are using nginx, then use configuration similar to -following: +For example if you are using nginx, then use configuration similar to following: server { listen 433 ssl; @@ -75,37 +67,28 @@ following: } } -Note the trailing slash on the `proxy_pass` directive, which causes nginx to -strip off the `/hydra/` part of the URL before passing it to hydra. +Note the trailing slash on the `proxy_pass` directive, which causes nginx to strip off the `/hydra/` part of the URL before passing it to hydra. Populating a Cache ------------------ -A common use for Hydra is to pre-build and cache derivations which -take a long time to build. While it is possible to direcly access the -Hydra server's store over SSH, a more scalable option is to upload -built derivations to a remote store like an [S3-compatible object -store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting -the `store_uri` parameter will cause Hydra to sign and upload -derivations as they are built: +A common use for Hydra is to pre-build and cache derivations which take a long time to build. +While it is possible to direcly access the Hydra server's store over SSH, a more scalable option is to upload built derivations to a remote store like an [S3-compatible object store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). +Setting the `store_uri` parameter will cause Hydra to sign and upload derivations as they are built: ``` store_uri = s3://cache-bucket-name?compression=zstd¶llel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key ``` -This example uses [Zstandard](https://github.com/facebook/zstd) -compression on derivations to reduce CPU usage on the server, but -[Brotli](https://brotli.org/) compression for derivation listings and -build logs because it has better browser support. +This example uses [Zstandard](https://github.com/facebook/zstd) compression on derivations to reduce CPU usage on the server, but [Brotli](https://brotli.org/) compression for derivation listings and build logs because it has better browser support. -See [`nix help -stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html) -for a description of the store URI format. +See [`nix help stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html) for a description of the store URI format. Statsd Configuration -------------------- -By default, Hydra will send stats to statsd at `localhost:8125`. Point Hydra to a different server via: +By default, Hydra will send stats to statsd at `localhost:8125`. +Point Hydra to a different server via: ``` @@ -117,9 +100,8 @@ By default, Hydra will send stats to statsd at `localhost:8125`. Point Hydra to hydra-notify's Prometheus service --------------------------------- -hydra-notify supports running a Prometheus webserver for metrics. The -exporter does not run unless a listen address and port are specified -in the hydra configuration file, as below: +hydra-notify supports running a Prometheus webserver for metrics. +The exporter does not run unless a listen address and port are specified in the hydra configuration file, as below: ```conf @@ -133,10 +115,9 @@ in the hydra configuration file, as below: hydra-queue-runner's Prometheus service --------------------------------------- -hydra-queue-runner supports running a Prometheus webserver for metrics. The -exporter's address defaults to exposing on `127.0.0.1:9198`, but is also -configurable through the hydra configuration file and a command line argument, -as below. A port of `:0` will make the exposer choose a random, available port. +hydra-queue-runner supports running a Prometheus webserver for metrics. +The exporter's address defaults to exposing on `127.0.0.1:9198`, but is also configurable through the hydra configuration file and a command line argument, as below. +A port of `:0` will make the exposer choose a random, available port. ```conf queue_runner_metrics_address = 127.0.0.1:9198 @@ -153,20 +134,17 @@ $ hydra-queue-runner --prometheus-address [::]:9198 Using LDAP as authentication backend (optional) ----------------------------------------------- -Instead of using Hydra's built-in user management you can optionally -use LDAP to manage roles and users. +Instead of using Hydra's built-in user management you can optionally use LDAP to manage roles and users. This is configured by defining the `` block in the configuration file. -In this block it's possible to configure the authentication plugin in the -`` block. All options are directly passed to `Catalyst::Authentication::Store::LDAP`. -The documentation for the available settings can be found -[here](https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS). +In this block it's possible to configure the authentication plugin in the `` block. +All options are directly passed to `Catalyst::Authentication::Store::LDAP`. +The documentation for the available settings can be found [here](https://metacpan.org/pod/Catalyst::Authentication::Store::LDAP#CONFIGURATION-OPTIONS). -Note that the bind password (if needed) should be supplied as an included file to -prevent it from leaking to the Nix store. +Note that the bind password (if needed) should be supplied as an included file to prevent it from leaking to the Nix store. -Roles can be assigned to users based on their LDAP group membership. For this -to work *use\_roles = 1* needs to be defined for the authentication plugin. +Roles can be assigned to users based on their LDAP group membership. +For this to work `use_roles = 1` needs to be defined for the authentication plugin. LDAP groups can then be mapped to Hydra roles using the `` block. Example configuration: @@ -244,12 +222,10 @@ Set the `debug` parameter under `ldap.config.ldap_server_options.debug`: ### Legacy LDAP Configuration -Hydra used to load the LDAP configuration from a YAML file in the -`HYDRA_LDAP_CONFIG` environment variable. This behavior is deperecated -and will be removed. +Hydra used to load the LDAP configuration from a YAML file in the `HYDRA_LDAP_CONFIG` environment variable. +This behavior is deperecated and will be removed. -When Hydra uses the deprecated YAML file, Hydra applies the following -default role mapping: +When Hydra uses the deprecated YAML file, Hydra applies the following default role mapping: ``` @@ -263,8 +239,7 @@ default role mapping: ``` -Note that configuring both the LDAP parameters in the hydra.conf and via -the environment variable is a fatal error. +Note that configuring both the LDAP parameters in the hydra.conf and via the environment variable is a fatal error. Webhook Authentication --------------------- diff --git a/subprojects/hydra-manual/src/hacking.md b/subprojects/hydra-manual/src/hacking.md index 689ff3177..80874bae8 100644 --- a/subprojects/hydra-manual/src/hacking.md +++ b/subprojects/hydra-manual/src/hacking.md @@ -1,15 +1,14 @@ # Hacking -This section provides some notes on how to hack on Hydra. To get the -latest version of Hydra from GitHub: +This section provides some notes on how to hack on Hydra. +To get the latest version of Hydra from GitHub: ```console $ git clone git://github.com/NixOS/hydra.git $ cd hydra ``` -To enter a shell in which all environment variables (such as `PERL5LIB`) -and dependencies can be found: +To enter a shell in which all environment variables (such as `PERL5LIB`) and dependencies can be found: ```console $ nix develop @@ -55,16 +54,12 @@ $ PERL5LIB=t/lib:$PERL5LIB perl t/test.pl t/Hydra/Controller/API/checks.t $ PERL5LIB=t/lib:$PERL5LIB perl t/test.pl t/Hydra/Controller/API/ ``` -**Warning**: Currently, the tests can fail -if run with high parallelism [due to an issue in -`Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40) -causing database ports to collide. +**Warning**: Currently, the tests can fail if run with high parallelism [due to an issue in `Test::PostgreSQL`](https://github.com/TJC/Test-postgresql/issues/40) causing database ports to collide. ## Working on the Manual -By default, `foreman start` runs mdbook in "watch" mode. mdbook listens -at [http://localhost:63332/](http://localhost:63332/), and -will reload the page every time you save. +By default, `foreman start` runs mdbook in "watch" mode. +mdbook listens at [http://localhost:63332/](http://localhost:63332/), and will reload the page every time you save. ## Building @@ -78,8 +73,7 @@ $ nix build .#packages.x86_64-linux.default ### Connecting to the database -Assuming you're running the default configuration with `foreman start`, -open an interactive session with Postgres via: +Assuming you're running the default configuration with `foreman start`, open an interactive session with Postgres via: ```console $ psql -h localhost -p 64444 hydra @@ -94,16 +88,14 @@ $ cargo build -p hydra-queue-runner $ cargo test -p hydra-queue-runner ``` -`foreman start` launches the queue-runner automatically with the right -config and environment. If a previous instance crashed, you may need to -remove the stale lock file: +`foreman start` launches the queue-runner automatically with the right config and environment. +If a previous instance crashed, you may need to remove the stale lock file: ```console $ rm .hydra-data/queue-runner/lock ``` -For `hydra-queue-runner` to successfully build locally, your -development user will need to be "trusted" by your Nix store. +For `hydra-queue-runner` to successfully build locally, your development user will need to be "trusted" by your Nix store. Add yourself to the `trusted_users` option of `/etc/nix/nix.conf`. diff --git a/subprojects/hydra-manual/src/installation.md b/subprojects/hydra-manual/src/installation.md index 39a86885d..1cdd99de6 100644 --- a/subprojects/hydra-manual/src/installation.md +++ b/subprojects/hydra-manual/src/installation.md @@ -1,92 +1,69 @@ Installation ============ -This chapter explains how to install Hydra on your own build farm -server. +This chapter explains how to install Hydra on your own build farm server. Prerequisites ------------- -To install and use Hydra you need to have installed the following -dependencies: +To install and use Hydra you need to have installed the following dependencies: - Nix - PostgreSQL -- many Perl packages, notably Catalyst, EmailSender, and NixPerl (see - the [Hydra expression in - Nixpkgs](https://github.com/NixOS/hydra/blob/master/release.nix) for - the complete list) +- many Perl packages, notably Catalyst, EmailSender, and NixPerl (see the [Hydra expression in Nixpkgs](https://github.com/NixOS/hydra/blob/master/release.nix) for the complete list) -At the moment, Hydra runs only on GNU/Linux (*i686-linux* and -*x86\_64\_linux*). +At the moment, Hydra runs only on GNU/Linux (*i686-linux* and *x86_64_linux*). For small projects, Hydra can be run on any reasonably modern machine. -For individual projects you can even run Hydra on a laptop. However, the -charm of a buildfarm server is usually that it operates without -disturbing the developer\'s working environment and can serve releases -over the internet. In conjunction you should typically have your source -code administered in a version management system, such as subversion. -Therefore, you will probably want to install a server that is connected -to the internet. To scale up to large and/or many projects, you will -need at least a considerable amount of diskspace to store builds. Since -Hydra can schedule multiple simultaneous build jobs, it can be useful to -have a multi-core machine, and/or attach multiple build machines in a -network to the central Hydra server. - -Of course we think it is a good idea to use the -[NixOS](http://nixos.org/nixos) GNU/Linux distribution for your -buildfarm server. But this is not a requirement. The Nix software -deployment system can be installed on any GNU/Linux distribution in -parallel to the regular package management system. Thus, you can use -Hydra on a Debian, Fedora, SuSE, or Ubuntu system. +For individual projects you can even run Hydra on a laptop. +However, the charm of a buildfarm server is usually that it operates without disturbing the developer's working environment and can serve releases over the internet. +In conjunction you should typically have your source code administered in a version management system, such as subversion. +Therefore, you will probably want to install a server that is connected to the internet. +To scale up to large and/or many projects, you will need at least a considerable amount of diskspace to store builds. +Since Hydra can schedule multiple simultaneous build jobs, it can be useful to have a multi-core machine, and/or attach multiple build machines in a network to the central Hydra server. + +Of course we think it is a good idea to use the [NixOS](http://nixos.org/nixos) GNU/Linux distribution for your buildfarm server. +But this is not a requirement. +The Nix software deployment system can be installed on any GNU/Linux distribution in parallel to the regular package management system. +Thus, you can use Hydra on a Debian, Fedora, SuSE, or Ubuntu system. Getting Nix ----------- -If your server runs NixOS you are all set to continue with installation -of Hydra. Otherwise you first need to install Nix. The latest stable -version can be found one [the Nix web -site](https://nixos.org/download/), along with a manual, which -includes installation instructions. +If your server runs NixOS you are all set to continue with installation of Hydra. +Otherwise you first need to install Nix. +The latest stable version can be found on [the Nix web site](https://nixos.org/download/), along with a manual, which includes installation instructions. Installation ------------ -The latest development snapshot of Hydra can be installed by visiting -the URL -[`http://hydra.nixos.org/view/hydra/unstable`](http://hydra.nixos.org/view/hydra/unstable) -and using the one-click install available at one of the build pages. You -can also install Hydra through the channel by performing the following -commands: +The latest development snapshot of Hydra can be installed by visiting the URL [`http://hydra.nixos.org/view/hydra/unstable`](http://hydra.nixos.org/view/hydra/unstable) and using the one-click install available at one of the build pages. +You can also install Hydra through the channel by performing the following commands: nix-channel --add http://hydra.nixos.org/jobset/hydra/master/channel/latest nix-channel --update nix-env -i hydra -Command completion should reveal a number of command-line tools from -Hydra, such as `hydra-queue-runner`. +Command completion should reveal a number of command-line tools from Hydra, such as `hydra-queue-runner`. Creating the database --------------------- Hydra stores its results in a PostgreSQL database. -To setup a PostgreSQL database with *hydra* as database name and user -name, issue the following commands on the PostgreSQL server: +To setup a PostgreSQL database with *hydra* as database name and user name, issue the following commands on the PostgreSQL server: ```console createuser -S -D -R -P hydra createdb -O hydra hydra ``` -Note that *\$prefix* is the location of Hydra in the nix store. +Note that *$prefix* is the location of Hydra in the nix store. -Hydra uses an environment variable to know which database should be -used, and a variable which point to a location that holds some state. To -set these variables for a PostgreSQL database, add the following to the -file `~/.profile` of the user running the Hydra services. +Hydra uses an environment variable to know which database should be used, and a variable which points to a location that holds some state. +To set these variables for a PostgreSQL database, add the following to the file `~/.profile` of the user running the Hydra services. ```console export HYDRA_DBI="dbi:Pg:dbname=hydra;host=dbserver.example.org;user=hydra;" @@ -99,11 +76,9 @@ You can provide the username and password in the file `~/.pgpass`, e.g. dbserver.example.org:*:hydra:hydra:password ``` -Make sure that the *HYDRA\_DATA* directory exists and is writable for -the user which will run the Hydra services. +Make sure that the *HYDRA_DATA* directory exists and is writable for the user which will run the Hydra services. -Having set these environment variables, you can now initialise the -database by doing: +Having set these environment variables, you can now initialise the database by doing: ```console hydra-init @@ -122,8 +97,7 @@ Additional users can be created through the web interface. Upgrading --------- -If you\'re upgrading Hydra from a previous version, you should do the -following to perform any necessary database schema migrations: +If you're upgrading Hydra from a previous version, you should do the following to perform any necessary database schema migrations: ```console hydra-init @@ -141,25 +115,12 @@ hydra-server When the server is started, you can browse to [http://localhost:3000/]() to start configuring your Hydra instance. -The `hydra-server` command launches the web server. There are two other -processes that come into play: - -- The - evaluator - is responsible for periodically evaluating job sets, checking out - their dependencies off their version control systems (VCS), and - queueing new builds if the result of the evaluation changed. It is - launched by the - hydra-evaluator - command. -- The - queue runner - launches builds (using Nix) as they are queued by the evaluator, - scheduling them onto the configured Nix hosts. It is launched using - the - hydra-queue-runner - command. - -All three processes must be running for Hydra to be fully functional, -though it\'s possible to temporarily stop any one of them for -maintenance purposes, for instance. +The `hydra-server` command launches the web server. +There are two other processes that come into play: + +- The *evaluator* is responsible for periodically evaluating job sets, checking out their dependencies off their version control systems (VCS), and queueing new builds if the result of the evaluation changed. + It is launched by the `hydra-evaluator` command. +- The *queue runner* launches builds (using Nix) as they are queued by the evaluator, scheduling them onto the configured Nix hosts. + It is launched using the `hydra-queue-runner` command. + +All three processes must be running for Hydra to be fully functional, though it's possible to temporarily stop any one of them for maintenance purposes, for instance. diff --git a/subprojects/hydra-manual/src/introduction.md b/subprojects/hydra-manual/src/introduction.md index b88f9b0f9..589ec1573 100644 --- a/subprojects/hydra-manual/src/introduction.md +++ b/subprojects/hydra-manual/src/introduction.md @@ -4,170 +4,98 @@ Introduction About Hydra ----------- -Hydra is a tool for continuous integration testing and software release -that uses a purely functional language to describe build jobs and their -dependencies. Continuous integration is a simple technique to improve -the quality of the software development process. An automated system -continuously or periodically checks out the source code of a project, -builds it, runs tests, and produces reports for the developers. Thus, -various errors that might accidentally be committed into the code base -are automatically caught. Such a system allows more in-depth testing -than what developers could feasibly do manually: - -- Portability testing - : The software may need to be built and tested on many different - platforms. It is infeasible for each developer to do this before - every commit. -- Likewise, many projects have very large test sets (e.g., regression - tests in a compiler, or stress tests in a DBMS) that can take hours - or days to run to completion. -- Many kinds of static and dynamic analyses can be performed as part - of the tests, such as code coverage runs and static analyses. -- It may also be necessary to build many different - variants - of the software. For instance, it may be necessary to verify that - the component builds with various versions of a compiler. -- Developers typically use incremental building to test their changes - (since a full build may take too long), but this is unreliable with - many build management tools (such as Make), i.e., the result of the - incremental build might differ from a full build. -- It ensures that the software can be built from the sources under - revision control. Users of version management systems such as CVS - and Subversion often forget to place source files under revision - control. -- The machines on which the continuous integration system runs ideally - provides a clean, well-defined build environment. If this - environment is administered through proper SCM techniques, then - builds produced by the system can be reproduced. In contrast, - developer work environments are typically not under any kind of SCM - control. -- In large projects, developers often work on a particular component - of the project, and do not build and test the composition of those - components (again since this is likely to take too long). To prevent - the phenomenon of \`\`big bang integration\'\', where components are - only tested together near the end of the development process, it is - important to test components together as soon as possible (hence - continuous integration - ). -- It allows software to be - released - by automatically creating packages that users can download and - install. To do this manually represents an often prohibitive amount - of work, as one may want to produce releases for many different - platforms: e.g., installers for Windows and Mac OS X, RPM or Debian - packages for certain Linux distributions, and so on. - -In its simplest form, a continuous integration tool sits in a loop -building and releasing software components from a version management -system. For each component, it performs the following tasks: - -- It obtains the latest version of the component\'s source code from - the version management system. -- It runs the component\'s build process (which presumably includes - the execution of the component\'s test set). -- It presents the results of the build (such as error logs and - releases) to the developers, e.g., by producing a web page. - -Examples of continuous integration tools include Jenkins, CruiseControl -Tinderbox, Sisyphus, Anthill and BuildBot. These tools have various -limitations. - -- They do not manage the - build environment - . The build environment consists of the dependencies necessary to - perform a build action, e.g., compilers, libraries, etc. Setting up - the environment is typically done manually, and without proper SCM - control (so it may be hard to reproduce a build at a later time). - Manual management of the environment scales poorly in the number of - configurations that must be supported. For instance, suppose that we - want to build a component that requires a certain compiler X. We - then have to go to each machine and install X. If we later need a - newer version of X, the process must be repeated all over again. An - ever worse problem occurs if there are conflicting, mutually - exclusive versions of the dependencies. Thus, simply installing the - latest version is not an option. Of course, we can install these - components in different directories and manually pass the - appropriate paths to the build processes of the various components. +Hydra is a tool for continuous integration testing and software release that uses a purely functional language to describe build jobs and their dependencies. +Continuous integration is a simple technique to improve the quality of the software development process. +An automated system continuously or periodically checks out the source code of a project, builds it, runs tests, and produces reports for the developers. +Thus, various errors that might accidentally be committed into the code base are automatically caught. +Such a system allows more in-depth testing than what developers could feasibly do manually: + +- Portability testing: + The software may need to be built and tested on many different platforms. + It is infeasible for each developer to do this before every commit. +- Likewise, many projects have very large test sets (e.g., regression tests in a compiler, or stress tests in a DBMS) that can take hours or days to run to completion. +- Many kinds of static and dynamic analyses can be performed as part of the tests, such as code coverage runs and static analyses. +- It may also be necessary to build many different variants of the software. + For instance, it may be necessary to verify that the component builds with various versions of a compiler. +- Developers typically use incremental building to test their changes (since a full build may take too long), but this is unreliable with many build management tools (such as Make), i.e., the result of the incremental build might differ from a full build. +- It ensures that the software can be built from the sources under revision control. + Users of version management systems such as CVS and Subversion often forget to place source files under revision control. +- The machines on which the continuous integration system runs ideally provides a clean, well-defined build environment. + If this environment is administered through proper SCM techniques, then builds produced by the system can be reproduced. + In contrast, developer work environments are typically not under any kind of SCM control. +- In large projects, developers often work on a particular component of the project, and do not build and test the composition of those components (again since this is likely to take too long). + To prevent the phenomenon of "big bang integration", where components are only tested together near the end of the development process, it is important to test components together as soon as possible (hence *continuous integration*). +- It allows software to be released by automatically creating packages that users can download and install. + To do this manually represents an often prohibitive amount of work, as one may want to produce releases for many different platforms: e.g., installers for Windows and Mac OS X, RPM or Debian packages for certain Linux distributions, and so on. + +In its simplest form, a continuous integration tool sits in a loop building and releasing software components from a version management system. +For each component, it performs the following tasks: + +- It obtains the latest version of the component's source code from the version management system. +- It runs the component's build process (which presumably includes the execution of the component's test set). +- It presents the results of the build (such as error logs and releases) to the developers, e.g., by producing a web page. + +Examples of continuous integration tools include Jenkins, CruiseControl, Tinderbox, Sisyphus, Anthill and BuildBot. +These tools have various limitations. + +- They do not manage the *build environment*. + The build environment consists of the dependencies necessary to perform a build action, e.g., compilers, libraries, etc. + Setting up the environment is typically done manually, and without proper SCM control (so it may be hard to reproduce a build at a later time). + Manual management of the environment scales poorly in the number of configurations that must be supported. + For instance, suppose that we want to build a component that requires a certain compiler X. + We then have to go to each machine and install X. + If we later need a newer version of X, the process must be repeated all over again. + An ever worse problem occurs if there are conflicting, mutually exclusive versions of the dependencies. + Thus, simply installing the latest version is not an option. + Of course, we can install these components in different directories and manually pass the appropriate paths to the build processes of the various components. But this is a rather tiresome and error-prone process. -- They do not easily support - variability in software systems - . A system may have a great deal of build-time variability: optional - functionality, whether to build a debug or production version, - different versions of dependencies, and so on. (For instance, the - Linux kernel now has over 2,600 build-time configuration switches.) - It is therefore important that a continuous integration tool can - easily select and test different instances from the configuration - space of the system to reveal problems, such as erroneous - interactions between features. In a continuous integration setting, - it is also useful to test different combinations of versions of - subsystems, e.g., the head revision of a component against stable - releases of its dependencies, and vice versa, as this can reveal - various integration problems. - -*Hydra*, is a continuous integration tool that solves these problems. It -is built on top of the [Nix package manager](http://nixos.org/nix/), -which has a purely functional language for describing package build -actions and their dependencies. This allows the build environment for -projects to be produced automatically and deterministically, and -variability in components to be expressed naturally using functions; and -as such is an ideal fit for a continuous build system. +- They do not easily support *variability in software systems*. + A system may have a great deal of build-time variability: optional functionality, whether to build a debug or production version, different versions of dependencies, and so on. + (For instance, the Linux kernel now has over 2,600 build-time configuration switches.) + It is therefore important that a continuous integration tool can easily select and test different instances from the configuration space of the system to reveal problems, such as erroneous interactions between features. + In a continuous integration setting, it is also useful to test different combinations of versions of subsystems, e.g., the head revision of a component against stable releases of its dependencies, and vice versa, as this can reveal various integration problems. + +*Hydra* is a continuous integration tool that solves these problems. +It is built on top of the [Nix package manager](http://nixos.org/nix/), which has a purely functional language for describing package build actions and their dependencies. +This allows the build environment for projects to be produced automatically and deterministically, and variability in components to be expressed naturally using functions; and as such is an ideal fit for a continuous build system. About Us -------- -Hydra is the successor of the Nix Buildfarm, which was developed in -tandem with the Nix software deployment system. Nix was originally -developed at the Department of Information and Computing Sciences, -Utrecht University by the TraCE project (2003-2008). The project was -funded by the Software Engineering Research Program Jacquard to improve -the support for variability in software systems. Funding for the -development of Nix and Hydra is now provided by the NIRICT LaQuSo Build -Farm project. +Hydra is the successor of the Nix Buildfarm, which was developed in tandem with the Nix software deployment system. +Nix was originally developed at the Department of Information and Computing Sciences, Utrecht University by the TraCE project (2003-2008). +The project was funded by the Software Engineering Research Program Jacquard to improve the support for variability in software systems. +Funding for the development of Nix and Hydra is now provided by the NIRICT LaQuSo Build Farm project. About this Manual ----------------- -This manual tells you how to install the Hydra buildfarm software on -your own server and how to operate that server using its web interface. +This manual tells you how to install the Hydra buildfarm software on your own server and how to operate that server using its web interface. License ------- -Hydra is free software: you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your -option) any later version. +Hydra is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -Hydra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the [GNU General Public -License](http://www.gnu.org/licenses/) for more details. +Hydra is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +See the [GNU General Public License](http://www.gnu.org/licenses/) for more details. Hydra at `nixos.org` -------------------- -The `nixos.org` installation of Hydra runs at -[`http://hydra.nixos.org/`](http://hydra.nixos.org/). That installation -is used to build software components from the [Nix](http://nixos.org), -[NixOS](http://nixos.org/nixos), [GNU](http://www.gnu.org/), -[Stratego/XT](http://strategoxt.org), and related projects. - -If you are one of the developers on those projects, it is likely that -you will be using the NixOS Hydra server in some way. If you need to -administer automatic builds for your project, you should pull the right -strings to get an account on the server. This manual will tell you how -to set up new projects and build jobs within those projects and write a -release.nix file to describe the build process of your project to Hydra. +The `nixos.org` installation of Hydra runs at [`http://hydra.nixos.org/`](http://hydra.nixos.org/). +That installation is used to build software components from the [Nix](http://nixos.org), [NixOS](http://nixos.org/nixos), [GNU](http://www.gnu.org/), [Stratego/XT](http://strategoxt.org), and related projects. + +If you are one of the developers on those projects, it is likely that you will be using the NixOS Hydra server in some way. +If you need to administer automatic builds for your project, you should pull the right strings to get an account on the server. +This manual will tell you how to set up new projects and build jobs within those projects and write a release.nix file to describe the build process of your project to Hydra. You can skip the next chapter. -If your project does not yet have automatic builds within the NixOS -Hydra server, it may actually be eligible. We are in the process of -setting up a large buildfarm that should be able to support open source -and academic software projects. Get in touch. +If your project does not yet have automatic builds within the NixOS Hydra server, it may actually be eligible. +We are in the process of setting up a large buildfarm that should be able to support open source and academic software projects. +Get in touch. Hydra on your own buildfarm --------------------------- -If you need to run your own Hydra installation, -[installation chapter](installation.md) explains how to download and install the -system on your own server. +If you need to run your own Hydra installation, the [installation chapter](installation.md) explains how to download and install the system on your own server. diff --git a/subprojects/hydra-manual/src/monitoring/README.md b/subprojects/hydra-manual/src/monitoring/README.md index 1f17a64db..c770b556a 100644 --- a/subprojects/hydra-manual/src/monitoring/README.md +++ b/subprojects/hydra-manual/src/monitoring/README.md @@ -15,19 +15,15 @@ $ curl --header "Accept: application/json" http://localhost:63333/queue-runner-s ## Notification Daemon -The `hydra-notify` process can expose Prometheus metrics for plugin execution. See -[hydra-notify's Prometheus service](../configuration.md#hydra-notifys-prometheus-service) -for details on enabling and configuring the exporter. +The `hydra-notify` process can expose Prometheus metrics for plugin execution. +See [hydra-notify's Prometheus service](../configuration.md#hydra-notifys-prometheus-service) for details on enabling and configuring the exporter. -The notification exporter exposes metrics on a per-plugin, per-event-type basis: execution -durations, frequency, successes, and failures. +The notification exporter exposes metrics on a per-plugin, per-event-type basis: execution durations, frequency, successes, and failures. ### Diagnostic Dump -The notification daemon can also dump its metrics to stderr whether or not the exporter -is configured. This is particularly useful for cases where metrics data is needed but the -exporter was not enabled. +The notification daemon can also dump its metrics to stderr whether or not the exporter is configured. +This is particularly useful for cases where metrics data is needed but the exporter was not enabled. -To trigger this diagnostic dump, send a Postgres notification with the -`hydra_notify_dump_metrics` channel and no payload. See -[Re-sending a notification](../notifications.md#re-sending-a-notification). +To trigger this diagnostic dump, send a Postgres notification with the `hydra_notify_dump_metrics` channel and no payload. +See [Re-sending a notification](../notifications.md#re-sending-a-notification). diff --git a/subprojects/hydra-manual/src/notifications.md b/subprojects/hydra-manual/src/notifications.md index f1203af81..5bc36ab37 100644 --- a/subprojects/hydra-manual/src/notifications.md +++ b/subprojects/hydra-manual/src/notifications.md @@ -1,16 +1,18 @@ # `hydra-notify` and Hydra's Notifications -Hydra uses a notification-based subsystem to implement some features and support plugin development. Notifications are sent to `hydra-notify`, which is responsible for dispatching each notification to each plugin. +Hydra uses a notification-based subsystem to implement some features and support plugin development. +Notifications are sent to `hydra-notify`, which is responsible for dispatching each notification to each plugin. Notifications are passed from `hydra-queue-runner` to `hydra-notify` through Postgres's `NOTIFY` and `LISTEN` feature. ## Notification Types -Note that the notification format is subject to change and should not be considered an API. Integrate with `hydra-notify` instead of listening directly. +Note that the notification format is subject to change and should not be considered an API. +Integrate with `hydra-notify` instead of listening directly. ### `cached_build_finished` -* **Payload:** Exactly two values, tab separated: The ID of the evaluation which contains the finished build, followed by the ID of the finished build. +* **Payload:** Exactly two values, tab separated: the ID of the evaluation which contains the finished build, followed by the ID of the finished build. * **When:** Issued directly after an evaluation completes, when that evaluation includes this finished build. * **Delivery Semantics:** At most once per evaluation. @@ -24,60 +26,79 @@ Note that the notification format is subject to change and should not be conside ### `build_queued` * **Payload:** Exactly one value, the ID of the build. -* **When:** Issued after the transaction inserting the build in to the database is committed. One notification is sent per new build. -* **Delivery Semantics:** Ephemeral. `hydra-notify` must be running to react to this event. No record of this event is stored. +* **When:** Issued after the transaction inserting the build in to the database is committed. + One notification is sent per new build. +* **Delivery Semantics:** Ephemeral. + `hydra-notify` must be running to react to this event. + No record of this event is stored. ### `build_started` * **Payload:** Exactly one value, the ID of the build. * **When:** Issued directly before building happens, and only if the derivation's outputs cannot be substituted. -* **Delivery Semantics:** Ephemeral. `hydra-notify` must be running to react to this event. No record of this event is stored. +* **Delivery Semantics:** Ephemeral. + `hydra-notify` must be running to react to this event. + No record of this event is stored. ### `step_finished` -* **Payload:** Three values, tab separated: The ID of the build which the step is part of, the step number, and the path on disk to the log file. -* **When:** Issued directly after a step completes, regardless of success. Is not issued if the step's derivation's outputs can be substituted. -* **Delivery Semantics:** Ephemeral. `hydra-notify` must be running to react to this event. No record of this event is stored. +* **Payload:** Three values, tab separated: the ID of the build which the step is part of, the step number, and the path on disk to the log file. +* **When:** Issued directly after a step completes, regardless of success. + Is not issued if the step's derivation's outputs can be substituted. +* **Delivery Semantics:** Ephemeral. + `hydra-notify` must be running to react to this event. + No record of this event is stored. ### `build_finished` -* **Payload:** At least one value, tab separated: The ID of the build which finished, followed by IDs of all of the builds which also depended upon this build. +* **Payload:** At least one value, tab separated: the ID of the build which finished, followed by IDs of all of the builds which also depended upon this build. * **When:** Issued directly after a build completes, regardless of success and substitutability. * **Delivery Semantics:** At least once. `hydra-notify` will call `buildFinished` for each plugin in two ways: -* The `builds` table's `notificationspendingsince` column stores when the build finished. On startup, `hydra-notify` will query all builds with a non-null `notificationspendingsince` value and treat each row as a received `build_finished` event. +* The `builds` table's `notificationspendingsince` column stores when the build finished. + On startup, `hydra-notify` will query all builds with a non-null `notificationspendingsince` value and treat each row as a received `build_finished` event. * Additionally, `hydra-notify` subscribes to `build_finished` events and processes them in real time. After processing, the row's `notificationspendingsince` column is set to null. -It is possible for subsequent deliveries of the same `build_finished` data to imply different outcomes. For example, if the build fails, is restarted, and then succeeds. In this scenario the `build_finished` events will be delivered at least twice, once for the failure and then once for the success. +It is possible for subsequent deliveries of the same `build_finished` data to imply different outcomes. +For example, if the build fails, is restarted, and then succeeds. +In this scenario the `build_finished` events will be delivered at least twice, once for the failure and then once for the success. ### `eval_started` * **Payload:** Exactly two values, tab separated: an opaque trace ID representing this evaluation, and the ID of the jobset. * **When:** At the beginning of the evaluation phase for the jobset, before any work is done. -* **Delivery Semantics:** Ephemeral. `hydra-notify` must be running to react to this event. No record of this event is stored. +* **Delivery Semantics:** Ephemeral. + `hydra-notify` must be running to react to this event. + No record of this event is stored. ### `eval_added` * **Payload:** Exactly three values, tab separated: an opaque trace ID representing this evaluation, the ID of the jobset, and the ID of the JobsetEval record. * **When:** After the evaluator fetches inputs and completes the evaluation successfully. -* **Delivery Semantics:** Ephemeral. `hydra-notify` must be running to react to this event. No record of this event is stored. +* **Delivery Semantics:** Ephemeral. + `hydra-notify` must be running to react to this event. + No record of this event is stored. ### `eval_cached` * **Payload:** Exactly three values: an opaque trace ID representing this evaluation, the ID of the jobset, and the ID of the previous identical evaluation. * **When:** After the evaluator fetches inputs, if none of the inputs changed. -* **Delivery Semantics:** Ephemeral. `hydra-notify` must be running to react to this event. No record of this event is stored. +* **Delivery Semantics:** Ephemeral. + `hydra-notify` must be running to react to this event. + No record of this event is stored. ### `eval_failed` * **Payload:** Exactly two values: an opaque trace ID representing this evaluation, and the ID of the jobset. -* **When:** After any fetching any input fails, or any other evaluation error occurs. -* **Delivery Semantics:** Ephemeral. `hydra-notify` must be running to react to this event. No record of this event is stored. +* **When:** After fetching any input fails, or any other evaluation error occurs. +* **Delivery Semantics:** Ephemeral. + `hydra-notify` must be running to react to this event. + No record of this event is stored. ## Development Notes diff --git a/subprojects/hydra-manual/src/plugins/RunCommand.md b/subprojects/hydra-manual/src/plugins/RunCommand.md index 652a171e1..4fba6b162 100644 --- a/subprojects/hydra-manual/src/plugins/RunCommand.md +++ b/subprojects/hydra-manual/src/plugins/RunCommand.md @@ -33,17 +33,18 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa ### Dynamic Commands -Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In -order to enable dynamic commands, you must enable this feature in your -`hydra.conf`, *as well as* in the parent project and jobset configuration. +Hydra can optionally run RunCommand hooks defined dynamically by the jobset. +In order to enable dynamic commands, you must enable this feature in your `hydra.conf`, *as well as* in the parent project and jobset configuration. #### Behavior -Hydra will execute any program defined under the `runCommandHook` attribute set. These jobs must have a single output named `out`, and that output must be an executable file located directly at `$out`. +Hydra will execute any program defined under the `runCommandHook` attribute set. +These jobs must have a single output named `out`, and that output must be an executable file located directly at `$out`. #### Security Properties -Safely deploying dynamic commands requires careful design of your Hydra jobs. Allowing arbitrary users to define attributes in your top level attribute set will allow that user to execute code on your Hydra. +Safely deploying dynamic commands requires careful design of your Hydra jobs. +Allowing arbitrary users to define attributes in your top level attribute set will allow that user to execute code on your Hydra. If a jobset has dynamic commands enabled, you must ensure only trusted users can define top level attributes. diff --git a/subprojects/hydra-manual/src/plugins/declarative-projects.md b/subprojects/hydra-manual/src/plugins/declarative-projects.md index b72c6fd08..ce908f69c 100644 --- a/subprojects/hydra-manual/src/plugins/declarative-projects.md +++ b/subprojects/hydra-manual/src/plugins/declarative-projects.md @@ -1,29 +1,26 @@ ## Declarative Projects -Hydra supports declaratively configuring a project\'s jobsets. This -configuration can be done statically, or generated by a build job. +Hydra supports declaratively configuring a project's jobsets. +This configuration can be done statically, or generated by a build job. > **Note** > -> Hydra will treat the project\'s declarative input as a static definition -> if and only if the spec file contains a dictionary of dictionaries. If -> the value of any key in the spec is not a dictionary, it will treat the -> spec as a generated declarative spec. +> Hydra will treat the project's declarative input as a static definition if and only if the spec file contains a dictionary of dictionaries. +> If the value of any key in the spec is not a dictionary, it will treat the spec as a generated declarative spec. ### Static, Declarative Projects -Hydra supports declarative projects, where jobsets are configured from a -static JSON document in a repository. +Hydra supports declarative projects, where jobsets are configured from a static JSON document in a repository. To configure a static declarative project, take the following steps: 1. Create a Hydra-fetchable source like a Git repository or local path. -2. In that source, create a file called `spec.json`, and add the - specification for all of the jobsets. Each key is jobset and each - value is a jobset\'s specification. For example: +2. In that source, create a file called `spec.json`, and add the specification for all of the jobsets. + Each key is jobset and each value is a jobset's specification. + For example: - ``` {.json} + ```json { "nixpkgs": { "enabled": 1, @@ -68,79 +65,54 @@ To configure a static declarative project, take the following steps: } ``` -3. Create a new project, and set the project\'s declarative input type, - declarative input value, and declarative spec file to point to the - source and JSON file you created in step 2. +3. Create a new project, and set the project's declarative input type, declarative input value, and declarative spec file to point to the source and JSON file you created in step 2. -Hydra will create a special jobset named `.jobsets`. When the `.jobsets` -jobset is evaluated, this static specification will be used for -configuring the rest of the project\'s jobsets. +Hydra will create a special jobset named `.jobsets`. +When the `.jobsets` jobset is evaluated, this static specification will be used for configuring the rest of the project's jobsets. ### Generated, Declarative Projects -Hydra also supports generated declarative projects, where jobsets are -configured automatically from specification files instead of being -managed through the UI. A jobset specification is a JSON object -containing the configuration of the jobset, for example: - -``` {.json} - { - "enabled": 1, - "hidden": false, - "description": "js", - "nixexprinput": "src", - "nixexprpath": "release.nix", - "checkinterval": 300, - "schedulingshares": 100, - "enableemail": false, - "enable_dynamic_run_command": false, - "emailoverride": "", - "keepnr": 3, - "inputs": { - "src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false }, - "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false } - } +Hydra also supports generated declarative projects, where jobsets are configured automatically from specification files instead of being managed through the UI. +A jobset specification is a JSON object containing the configuration of the jobset, for example: + +```json +{ + "enabled": 1, + "hidden": false, + "description": "js", + "nixexprinput": "src", + "nixexprpath": "release.nix", + "checkinterval": 300, + "schedulingshares": 100, + "enableemail": false, + "enable_dynamic_run_command": false, + "emailoverride": "", + "keepnr": 3, + "inputs": { + "src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false }, + "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false } } - +} ``` To configure a declarative project, take the following steps: -1. Create a jobset repository in the normal way (e.g. a git repo with a - `release.nix` file, any other needed helper files, and taking any - kind of hydra input), but without adding it to the UI. The nix - expression of this repository should contain a single job, named - `jobsets`. The output of the `jobsets` job should be a JSON file - containing an object of jobset specifications. Each member of the - object will become a jobset of the project, configured by the - corresponding jobset specification. - -2. In some hydra-fetchable source (potentially, but not necessarily, - the same repo you created in step 1), create a JSON file containing - a jobset specification that points to the jobset repository you - created in the first step, specifying any needed inputs - (e.g. nixpkgs) as necessary. - -3. In the project creation/edit page, set declarative input type, - declarative input value, and declarative spec file to point to the - source and JSON file you created in step 2. - -Hydra will create a special jobset named `.jobsets`, which whenever -evaluated will go through the steps above in reverse order: - -1. Hydra will fetch the input specified by the declarative input type - and value. - -2. Hydra will use the configuration given in the declarative spec file - as the jobset configuration for this evaluation. In addition to any - inputs specified in the spec file, hydra will also pass the - `declInput` argument corresponding to the input fetched in step 1 and - the `projectName` argument containing the project\'s name. - -3. As normal, hydra will build the jobs specified in the jobset - repository, which in this case is the single `jobsets` job. When - that job completes, hydra will read the created jobset - specifications and create corresponding jobsets in the project, - disabling any jobsets that used to exist but are not present in the - current spec. +1. Create a jobset repository in the normal way (e.g. a git repo with a `release.nix` file, any other needed helper files, and taking any kind of hydra input), but without adding it to the UI. + The nix expression of this repository should contain a single job, named `jobsets`. + The output of the `jobsets` job should be a JSON file containing an object of jobset specifications. + Each member of the object will become a jobset of the project, configured by the corresponding jobset specification. + +2. In some hydra-fetchable source (potentially, but not necessarily, the same repo you created in step 1), create a JSON file containing a jobset specification that points to the jobset repository you created in the first step, specifying any needed inputs (e.g. nixpkgs) as necessary. + +3. In the project creation/edit page, set declarative input type, declarative input value, and declarative spec file to point to the source and JSON file you created in step 2. + +Hydra will create a special jobset named `.jobsets`, which whenever evaluated will go through the steps above in reverse order: + +1. Hydra will fetch the input specified by the declarative input type and value. + +2. Hydra will use the configuration given in the declarative spec file as the jobset configuration for this evaluation. + In addition to any inputs specified in the spec file, hydra will also pass the `declInput` argument corresponding to the input fetched in step 1 and the `projectName` argument containing the project's name. + +3. As normal, hydra will build the jobs specified in the jobset repository, which in this case is the single `jobsets` job. + When that job completes, hydra will read the created jobset specifications and create corresponding jobsets in the project, disabling any jobsets that used to exist but are not present in the current spec. diff --git a/subprojects/hydra-manual/src/projects.md b/subprojects/hydra-manual/src/projects.md index f7c4975fc..b25df6564 100644 --- a/subprojects/hydra-manual/src/projects.md +++ b/subprojects/hydra-manual/src/projects.md @@ -1,35 +1,28 @@ Creating and Managing Projects ============================== -Once Hydra is installed and running, the next step is to add projects to -the build farm. We follow the example of the [Patchelf -project](http://nixos.org/patchelf.html), a software tool written in C -and using the GNU Build System (GNU Autoconf and GNU Automake). +Once Hydra is installed and running, the next step is to add projects to the build farm. +We follow the example of the [Patchelf project](http://nixos.org/patchelf.html), a software tool written in C and using the GNU Build System (GNU Autoconf and GNU Automake). -Log in to the web interface of your Hydra installation using the user -name and password you inserted in the database (by default, Hydra\'s web -server listens on [`localhost:3000`](http://localhost:3000/)). Then -follow the \"Create Project\" link to create a new project. +Log in to the web interface of your Hydra installation using the user name and password you inserted in the database (by default, Hydra's web server listens on [`localhost:3000`](http://localhost:3000/)). +Then follow the "Create Project" link to create a new project. Project Information ------------------- -A project definition consists of some general information and a set of -job sets. The general information identifies a project, its owner, and -current state of activity. Here\'s what we fill in for the patchelf -project: +A project definition consists of some general information and a set of job sets. +The general information identifies a project, its owner, and current state of activity. +Here's what we fill in for the patchelf project: Identifier: patchelf -The *identifier* is the identity of the project. It is used in URLs and -in the names of build results. +The *identifier* is the identity of the project. +It is used in URLs and in the names of build results. -The identifier should be a unique name (it is the primary database key -for the project table in the database). If you try to create a project -with an already existing identifier you\'d get an error message from the -database. So try to create the project after entering just the general -information to figure out if you have chosen a unique name. Job sets can -be added once the project has been created. +The identifier should be a unique name (it is the primary database key for the project table in the database). +If you try to create a project with an already existing identifier you'd get an error message from the database. +So try to create the project after entering just the general information to figure out if you have chosen a unique name. +Job sets can be added once the project has been created. Display name: Patchelf @@ -37,8 +30,7 @@ The *display name* is used in menus. Description: A tool for modifying ELF binaries -The *description* is used as short documentation of the nature of the -project. +The *description* is used as short documentation of the nature of the project. Owner: eelco @@ -48,31 +40,24 @@ The *owner* of a project can create and edit job sets. Only if the project is *enabled* are builds performed. -Once created there should be an entry for the project in the sidebar. Go -to the project page for the -[Patchelf](http://localhost:3000/project/patchelf) project. +Once created there should be an entry for the project in the sidebar. +Go to the project page for the [Patchelf](http://localhost:3000/project/patchelf) project. Job Sets -------- -A project can consist of multiple *job sets* (hereafter *jobsets*), -separate tasks that can be built separately, but may depend on each -other (without cyclic dependencies, of course). Go to the -[Edit](http://localhost:3000/project/patchelf/edit) page of the Patchelf -project and \"Add a new jobset\" by providing the following -\"Information\": +A project can consist of multiple *job sets* (hereafter *jobsets*), separate tasks that can be built separately, but may depend on each other (without cyclic dependencies, of course). +Go to the [Edit](http://localhost:3000/project/patchelf/edit) page of the Patchelf project and "Add a new jobset" by providing the following "Information": Identifier: trunk Description: Trunk Nix expression: release.nix in input patchelfSrc -This states that in order to build the `trunk` jobset, the Nix -expression in the file `release.nix`, which can be obtained from input -`patchelfSrc`, should be evaluated. (We\'ll have a look at `release.nix` -later.) +This states that in order to build the `trunk` jobset, the Nix expression in the file `release.nix`, which can be obtained from input `patchelfSrc`, should be evaluated. +(We'll have a look at `release.nix` later.) -To realize a job we probably need a number of inputs, which can be -declared in the table below. As many inputs as required can be added. +To realize a job we probably need a number of inputs, which can be declared in the table below. +As many inputs as required can be added. For patchelf we declare the following inputs. patchelfSrc @@ -90,147 +75,106 @@ Building Jobs Build Recipes ------------- -Build jobs and *build recipes* for a jobset are specified in a text file -written in the [Nix language](http://nixos.org/nix/). The recipe is -actually called a *Nix expression* in Nix parlance. By convention this -file is often called `release.nix`. - -The `release.nix` file is typically kept under version control, and the -repository that contains it one of the build inputs of the -corresponding--often called `hydraConfig` by convention. The repository -for that file and the actual file name are specified on the web -interface of Hydra under the `Setup` tab of the jobset\'s overview page, -under the `Nix - expression` heading. See, for example, the [jobset overview -page](http://hydra.nixos.org/jobset/patchelf/trunk) of the PatchELF -project, and [the corresponding Nix -file](https://github.com/NixOS/patchelf/blob/master/release.nix). - -Knowledge of the Nix language is recommended, but the example below -should already give a good idea of how it works: - - let - pkgs = import {}; â‘  - - jobs = rec { â‘¡ - - tarball = â‘¢ - pkgs.releaseTools.sourceTarball { â‘£ - name = "hello-tarball"; - src = ; ⑤ - buildInputs = (with pkgs; [ gettext texLive texinfo ]); - }; - - build = â‘¥ - { system ? builtins.currentSystem }: ⑦ - - let pkgs = import { inherit system; }; in - pkgs.releaseTools.nixBuild { â‘§ - name = "hello"; - src = jobs.tarball; - configureFlags = [ "--disable-silent-rules" ]; - }; +Build jobs and *build recipes* for a jobset are specified in a text file written in the [Nix language](http://nixos.org/nix/). +The recipe is actually called a *Nix expression* in Nix parlance. +By convention this file is often called `release.nix`. + +The `release.nix` file is typically kept under version control, and the repository that contains it one of the build inputs of the corresponding — often called `hydraConfig` by convention. +The repository for that file and the actual file name are specified on the web interface of Hydra under the `Setup` tab of the jobset's overview page, under the `Nix expression` heading. +See, for example, the [jobset overview page](http://hydra.nixos.org/jobset/patchelf/trunk) of the PatchELF project, and [the corresponding Nix file](https://github.com/NixOS/patchelf/blob/master/release.nix). + +Knowledge of the Nix language is recommended, but the example below should already give a good idea of how it works: + +```nix +let + pkgs = import {}; # â‘  + + jobs = rec { # â‘¡ + + tarball = # â‘¢ + pkgs.releaseTools.sourceTarball { # â‘£ + name = "hello-tarball"; + src = ; # ⑤ + buildInputs = (with pkgs; [ gettext texLive texinfo ]); + }; + + build = # â‘¥ + { system ? builtins.currentSystem }: # ⑦ + + let pkgs = import { inherit system; }; in + pkgs.releaseTools.nixBuild { # â‘§ + name = "hello"; + src = jobs.tarball; + configureFlags = [ "--disable-silent-rules" ]; }; - in - jobs ⑨ - - -This file shows what a `release.nix` file for -[GNU Hello](http://www.gnu.org/software/hello/) would look like. -GNU Hello is representative of many GNU and non-GNU free software -projects: - -- it uses the GNU Build System, namely GNU Autoconf, and GNU Automake; - for users, it means it can be installed using the - usual - ./configure && make install - procedure - ; + }; +in + jobs # ⑨ +``` + +This file shows what a `release.nix` file for [GNU Hello](http://www.gnu.org/software/hello/) would look like. +GNU Hello is representative of many GNU and non-GNU free software projects: + +- it uses the GNU Build System, namely GNU Autoconf, and GNU Automake; + for users, it means it can be installed using the usual `./configure && make install` procedure; - it uses Gettext for internationalization; - it has a Texinfo manual, which can be rendered as PDF with TeX. -The file defines a jobset consisting of two jobs: `tarball`, and -`build`. It contains the following elements (referenced from the figure -by numbers): - -1. This defines a variable `pkgs` holding the set of packages provided - by [Nixpkgs](http://nixos.org/nixpkgs/). - - Since `nixpkgs` appears in angle brackets, there must be a build - input of that name in the Nix search path. In this case, the web - interface should show a `nixpkgs` build input, which is a checkout - of the Nixpkgs source code repository; Hydra then adds this and - other build inputs to the Nix search path when evaluating - `release.nix`. - -2. This defines a variable holding the two Hydra jobs--an *attribute - set* in Nix. - -3. This is the definition of the first job, named `tarball`. The - purpose of this job is to produce a usable source code tarball. - -4. The `tarball` job calls the `sourceTarball` function, which - (roughly) runs `autoreconf && ./configure && - make dist` on the checkout. The `buildInputs` attribute - specifies additional software dependencies for the job. - - > The package names used in `buildInputs`--e.g., `texLive`--are the - > names of the *attributes* corresponding to these packages in - > Nixpkgs, specifically in the - > [`all-packages.nix`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/all-packages.nix) - > file. See the section entitled "Package Naming" in the Nixpkgs - > manual for more information. - -5. The `tarball` jobs expects a `hello` build input to be available in - the Nix search path. Again, this input is passed by Hydra and is - meant to be a checkout of GNU Hello\'s source code repository. - -6. This is the definition of the `build` job, whose purpose is to build - Hello from the tarball produced above. - -7. The `build` function takes one parameter, `system`, which should be - a string defining the Nix system type--e.g., `"x86_64-linux"`. +The file defines a jobset consisting of two jobs: `tarball`, and `build`. +It contains the following elements (referenced from the figure by numbers): + +1. This defines a variable `pkgs` holding the set of packages provided by [Nixpkgs](http://nixos.org/nixpkgs/). + + Since `nixpkgs` appears in angle brackets, there must be a build input of that name in the Nix search path. + In this case, the web interface should show a `nixpkgs` build input, which is a checkout of the Nixpkgs source code repository; Hydra then adds this and other build inputs to the Nix search path when evaluating `release.nix`. + +2. This defines a variable holding the two Hydra jobs -- an *attribute set* in Nix. + +3. This is the definition of the first job, named `tarball`. + The purpose of this job is to produce a usable source code tarball. + +4. The `tarball` job calls the `sourceTarball` function, which (roughly) runs `autoreconf && ./configure && make dist` on the checkout. + The `buildInputs` attribute specifies additional software dependencies for the job. + + > The package names used in `buildInputs` -- e.g., `texLive` -- are the names of the *attributes* corresponding to these packages in Nixpkgs, specifically in the [`all-packages.nix`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/all-packages.nix) file. + > See the section entitled "Package Naming" in the Nixpkgs manual for more information. + +5. The `tarball` jobs expects a `hello` build input to be available in the Nix search path. + Again, this input is passed by Hydra and is meant to be a checkout of GNU Hello's source code repository. + +6. This is the definition of the `build` job, whose purpose is to build Hello from the tarball produced above. + +7. The `build` function takes one parameter, `system`, which should be a string defining the Nix system type -- e.g., `"x86_64-linux"`. Additionally, it refers to `jobs.tarball`, seen above. - Hydra inspects the formal argument list of the function (here, the - `system` argument) and passes it the corresponding parameter - specified as a build input on Hydra\'s web interface. Here, `system` - is passed by Hydra when it calls `build`. Thus, it must be defined - as a build input of type string in Hydra, which could take one of - several values. + Hydra inspects the formal argument list of the function (here, the `system` argument) and passes it the corresponding parameter specified as a build input on Hydra's web interface. + Here, `system` is passed by Hydra when it calls `build`. + Thus, it must be defined as a build input of type string in Hydra, which could take one of several values. - The question mark after `system` defines the default value for this - argument, and is only useful when debugging locally. + The question mark after `system` defines the default value for this argument, and is only useful when debugging locally. -8. The `build` job calls the `nixBuild` function, which unpacks the - tarball, then runs `./configure && make - && make check && make install`. +8. The `build` job calls the `nixBuild` function, which unpacks the tarball, then runs `./configure && make && make check && make install`. -9. Finally, the set of jobs is returned to Hydra, as a Nix attribute - set. +9. Finally, the set of jobs is returned to Hydra, as a Nix attribute set. Building from the Command Line ------------------------------ -It is often useful to test a build recipe, for instance before it is -actually used by Hydra, when testing changes, or when debugging a build -issue. Since build recipes for Hydra jobsets are just plain Nix -expressions, they can be evaluated using the standard Nix tools. +It is often useful to test a build recipe, for instance before it is actually used by Hydra, when testing changes, or when debugging a build issue. +Since build recipes for Hydra jobsets are just plain Nix expressions, they can be evaluated using the standard Nix tools. -To evaluate the `tarball` jobset of the above example, just -run: +To evaluate the `tarball` jobset of the above example, just run: ```console $ nix-build release.nix -A tarball ``` -However, doing this with the example as is will probably -yield an error like this: +However, doing this with the example as is will probably yield an error like this: error: user-thrown exception: file `hello' was not found in the Nix search path (add it using $NIX_PATH or -I) -The error is self-explanatory. Assuming `$HOME/src/hello` points to a -checkout of Hello, this can be fixed this way: +The error is self-explanatory. +Assuming `$HOME/src/hello` points to a checkout of Hello, this can be fixed this way: ```console $ nix-build -I ~/src release.nix -A tarball @@ -242,22 +186,16 @@ Similarly, the `build` jobset can be evaluated: $ nix-build -I ~/src release.nix -A build ``` -The `build` job reuses the result of the `tarball` job, rebuilding it -only if it needs to. +The `build` job reuses the result of the `tarball` job, rebuilding it only if it needs to. Adding More Jobs ---------------- -The example illustrates how to write the most basic -jobs, `tarball` and `build`. In practice, much more can be done by using -features readily provided by Nixpkgs or by creating new jobs as -customizations of existing jobs. +The example illustrates how to write the most basic jobs, `tarball` and `build`. +In practice, much more can be done by using features readily provided by Nixpkgs or by creating new jobs as customizations of existing jobs. -For instance, test coverage report for projects compiled with GCC can be -automatically generated using the `coverageAnalysis` function provided -by Nixpkgs instead of `nixBuild`. Back to our GNU Hello example, we can -define a `coverage` job that produces an HTML code coverage report -directly readable from the corresponding Hydra build page: +For instance, test coverage report for projects compiled with GCC can be automatically generated using the `coverageAnalysis` function provided by Nixpkgs instead of `nixBuild`. +Back to our GNU Hello example, we can define a `coverage` job that produces an HTML code coverage report directly readable from the corresponding Hydra build page: coverage = { system ? builtins.currentSystem }: @@ -269,20 +207,14 @@ directly readable from the corresponding Hydra build page: configureFlags = [ "--disable-silent-rules" ]; }; -As can be seen, the only difference compared to `build` is the use of -`coverageAnalysis`. +As can be seen, the only difference compared to `build` is the use of `coverageAnalysis`. -Nixpkgs provides many more build tools, including the ability to run -build in virtual machines, which can themselves run another GNU/Linux -distribution, which allows for the creation of packages for these -distributions. Please see [the `pkgs/build-support/release` -directory](https://github.com/NixOS/nixpkgs/tree/master/pkgs/build-support/release) -of Nixpkgs for more. The NixOS manual also contains information about -whole-system testing in virtual machine. +Nixpkgs provides many more build tools, including the ability to run build in virtual machines, which can themselves run another GNU/Linux distribution, which allows for the creation of packages for these distributions. +Please see [the `pkgs/build-support/release` directory](https://github.com/NixOS/nixpkgs/tree/master/pkgs/build-support/release) of Nixpkgs for more. +The NixOS manual also contains information about whole-system testing in virtual machine. -Now, assume we want to build Hello with an old version of GCC, and with -different `configure` flags. A new `build_exotic` job can be written -that simply *overrides* the relevant arguments passed to `nixBuild`: +Now, assume we want to build Hello with an old version of GCC, and with different `configure` flags. +A new `build_exotic` job can be written that simply *overrides* the relevant arguments passed to `nixBuild`: build_exotic = { system ? builtins.currentSystem }: @@ -298,14 +230,11 @@ that simply *overrides* the relevant arguments passed to `nixBuild`: attrs.configureFlags ++ [ "--disable-nls" ]; }); -The `build_exotic` job reuses `build` and overrides some of its -arguments: it adds a dependency on GCC 3.3, a pre-configure phase that -runs `gcc --version`, and adds the `--disable-nls` configure flags. +The `build_exotic` job reuses `build` and overrides some of its arguments: it adds a dependency on GCC 3.3, a pre-configure phase that runs `gcc --version`, and adds the `--disable-nls` configure flags. -This customization mechanism is very powerful. For instance, it can be -used to change the way Hello and *all* its dependencies--including the C -library and compiler used to build it--are built. See the Nixpkgs manual -for more. +This customization mechanism is very powerful. +For instance, it can be used to change the way Hello and *all* its dependencies -- including the C library and compiler used to build it -- are built. +See the Nixpkgs manual for more. Declarative Projects -------------------- @@ -316,8 +245,7 @@ Email Notifications ------------------- Hydra can send email notifications when the status of a build changes. -This provides immediate feedback to maintainers or committers when a -change causes build failures. +This provides immediate feedback to maintainers or committers when a change causes build failures. The feature can be turned on by adding the following line to `hydra.conf` @@ -325,17 +253,14 @@ The feature can be turned on by adding the following line to `hydra.conf` email_notification = 1 ``` -By default, Hydra only sends email notifications if a previously successful -build starts to fail. In order to force Hydra to send an email for each build -(including e.g. successful or cancelled ones), the environment variable -`HYDRA_FORCE_SEND_MAIL` can be declared: +By default, Hydra only sends email notifications if a previously successful build starts to fail. +In order to force Hydra to send an email for each build (including e.g. successful or cancelled ones), the environment variable `HYDRA_FORCE_SEND_MAIL` can be declared: ``` nix services.hydra-dev.extraEnv.HYDRA_FORCE_SEND_MAIL = "1"; ``` -SASL Authentication for the email address that's used to send notifications -can be configured like this: +SASL Authentication for the email address that's used to send notifications can be configured like this: ``` conf EMAIL_SENDER_TRANSPORT_sasl_username=hydra@example.org @@ -344,12 +269,10 @@ EMAIL_SENDER_TRANSPORT_port=587 EMAIL_SENDER_TRANSPORT_ssl=starttls ``` -Further information about these environment variables can be found at the -[MetaCPAN documentation of `Email::Sender::Manual::QuickStart`](https://metacpan.org/pod/Email::Sender::Manual::QuickStart#specifying-transport-in-the-environment). +Further information about these environment variables can be found at the [MetaCPAN documentation of `Email::Sender::Manual::QuickStart`](https://metacpan.org/pod/Email::Sender::Manual::QuickStart#specifying-transport-in-the-environment). -It's recommended to not put this in `services.hydra-dev.extraEnv` as this would -leak the secrets into the Nix store. Instead, it should be written into an -environment file and configured like this: +It's recommended to not put this in `services.hydra-dev.extraEnv` as this would leak the secrets into the Nix store. +Instead, it should be written into an environment file and configured like this: ``` nix { systemd.services.hydra-notify = { @@ -358,12 +281,9 @@ environment file and configured like this: } ``` -The simplest approach to enable Email Notifications is to use the ssmtp -package, which simply hands off the emails to another SMTP server. For -details on how to configure ssmtp, see the documentation for the -`networking.defaultMailServer` option. To use ssmtp for the Hydra email -notifications, add it to the path option of the Hydra services in your -`/etc/nixos/configuration.nix` file: +The simplest approach to enable Email Notifications is to use the ssmtp package, which simply hands off the emails to another SMTP server. +For details on how to configure ssmtp, see the documentation for the `networking.defaultMailServer` option. +To use ssmtp for the Hydra email notifications, add it to the path option of the Hydra services in your `/etc/nixos/configuration.nix` file: systemd.services.hydra-queue-runner.path = [ pkgs.ssmtp ]; systemd.services.hydra-server.path = [ pkgs.ssmtp ]; @@ -371,11 +291,9 @@ notifications, add it to the path option of the Hydra services in your Gitea Integration ----------------- -Hydra can notify Git servers (such as [GitLab](https://gitlab.com/), [GitHub](https://github.com) -or [Gitea](https://gitea.io/en-us/)) about the result of a build from a Git checkout. +Hydra can notify Git servers (such as [GitLab](https://gitlab.com/), [GitHub](https://github.com) or [Gitea](https://gitea.io/en-us/)) about the result of a build from a Git checkout. -This section describes how it can be implemented for `gitea`, but the approach for `gitlab` is -analogous: +This section describes how it can be implemented for `gitea`, but the approach for `gitlab` is analogous: * [Obtain an API token for your user](https://docs.gitea.io/en-us/api-usage/#authentication) * Add it to a file which only users in the hydra group can read like this: see [including files](configuration.md#including-files) for more information @@ -394,8 +312,7 @@ analogous: } ``` -* For a jobset with a `Git`-input which points to a `gitea`-instance, add the following - additional inputs: +* For a jobset with a `Git`-input which points to a `gitea`-instance, add the following additional inputs: | Type | Name | Value | | -------------- | ------------------- | ---------------------------------- | diff --git a/subprojects/hydra-manual/src/webhooks.md b/subprojects/hydra-manual/src/webhooks.md index 7a2117885..6afb14b73 100644 --- a/subprojects/hydra-manual/src/webhooks.md +++ b/subprojects/hydra-manual/src/webhooks.md @@ -1,12 +1,11 @@ # Webhooks -Hydra can be notified by github or gitea with webhooks to trigger a new evaluation when a -jobset has a github repo in its input. +Hydra can be notified by github or gitea with webhooks to trigger a new evaluation when a jobset has a github repo in its input. ## Webhook Authentication -Hydra supports webhook signature verification for both GitHub and Gitea using HMAC-SHA256. This ensures that webhook -requests are coming from your configured Git forge and haven't been tampered with. +Hydra supports webhook signature verification for both GitHub and Gitea using HMAC-SHA256. +This ensures that webhook requests are coming from your configured Git forge and haven't been tampered with. ### Configuring Webhook Authentication @@ -69,13 +68,12 @@ Then add the hook with `Add webhook`. ### Verifying GitHub Webhook Security -After configuration, GitHub will send webhook requests with an `X-Hub-Signature-256` header containing the HMAC-SHA256 -signature of the request body. Hydra will verify this signature matches the configured secret. +After configuration, GitHub will send webhook requests with an `X-Hub-Signature-256` header containing the HMAC-SHA256 signature of the request body. +Hydra will verify this signature matches the configured secret. ## Gitea -To set up a webhook for a Gitea repository go to the settings of the repository in your Gitea instance -and in the `Webhooks` tab click on `Add Webhook` and choose `Gitea` in the drop down. +To set up a webhook for a Gitea repository go to the settings of the repository in your Gitea instance and in the `Webhooks` tab click on `Add Webhook` and choose `Gitea` in the drop down. - In `Target URL` fill in `https:///api/push-gitea`. - Keep HTTP method `POST`, POST Content Type `application/json` and Trigger On `Push Events`. @@ -86,8 +84,8 @@ Then add the hook with `Add webhook`. ### Verifying Gitea Webhook Security -After configuration, Gitea will send webhook requests with an `X-Gitea-Signature` header containing the HMAC-SHA256 -signature of the request body. Hydra will verify this signature matches the configured secret. +After configuration, Gitea will send webhook requests with an `X-Gitea-Signature` header containing the HMAC-SHA256 signature of the request body. +Hydra will verify this signature matches the configured secret. ## Troubleshooting diff --git a/subprojects/hydra-queue-runner/Cargo.toml b/subprojects/hydra-queue-runner/Cargo.toml index d5b513bce..177d22d04 100644 --- a/subprojects/hydra-queue-runner/Cargo.toml +++ b/subprojects/hydra-queue-runner/Cargo.toml @@ -18,10 +18,10 @@ serde_json.workspace = true smallvec = { workspace = true, features = [ "serde" ] } toml.workspace = true -anyhow.workspace = true atomic_float.workspace = true backon.workspace = true clap = { workspace = true, features = [ "derive", "env" ] } +color-eyre.workspace = true fs-err = { workspace = true, features = [ "tokio" ] } thiserror.workspace = true uuid = { workspace = true, features = [ "v4", "serde" ] } @@ -52,20 +52,25 @@ prometheus = { workspace = true, features = [ "process" ] } tower = { workspace = true, features = [ "util" ] } tower-http = { workspace = true, features = [ "trace" ] } -binary-cache = { path = "../crates/binary-cache" } -db = { path = "../crates/db" } -harmonia-store-core.workspace = true -hydra-tracing = { path = "../crates/tracing", features = [ "tonic" ] } -nix-utils = { path = "../crates/nix-utils" } -shared = { path = "../crates/shared" } +binary-cache.workspace = true +daemon-client-utils.workspace = true +db.workspace = true +harmonia-protocol.workspace = true +harmonia-store-aterm.workspace = true +harmonia-store-content-address.workspace = true +harmonia-store-derivation.workspace = true +harmonia-store-nar-info.workspace = true +harmonia-store-path.workspace = true +harmonia-store-path-info.workspace = true +harmonia-store-remote.workspace = true +harmonia-utils-hash.workspace = true +hydra-proto = { workspace = true, features = [ "db", "server" ] } +hydra-tracing = { workspace = true, features = [ "tonic" ] } +nix-support.workspace = true +store-transfer.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator.workspace = true -[build-dependencies] -fs-err = { workspace = true } -sha2.workspace = true -tonic-prost-build.workspace = true - [features] otel = [ "hydra-tracing/otel" ] diff --git a/subprojects/hydra-queue-runner/build.rs b/subprojects/hydra-queue-runner/build.rs deleted file mode 100644 index 2183eceff..000000000 --- a/subprojects/hydra-queue-runner/build.rs +++ /dev/null @@ -1,31 +0,0 @@ -use sha2::Digest; -use std::{env, path::PathBuf}; - -fn main() -> Result<(), Box> { - let out_dir = PathBuf::from(env::var("OUT_DIR")?); - - let workspace_version = env::var("CARGO_PKG_VERSION")?; - - let proto_path = "../proto/v1/streaming.proto"; - let proto_content = fs_err::read_to_string(proto_path)?; - let mut hasher = sha2::Sha256::new(); - hasher.update(proto_content.as_bytes()); - let proto_hash = format!("{:x}", hasher.finalize()); - let version = format!("{}-{}", workspace_version, &proto_hash[..8]); - - // Generate version module - fs_err::write( - out_dir.join("proto_version.rs"), - format!( - r#"// Generated during build - do not edit -pub const PROTO_API_VERSION: &str = "{version}"; -"# - ), - )?; - - tonic_prost_build::configure() - .extern_path(".runner.v1.StorePath", "::shared::proto::ProtoStorePath") - .file_descriptor_set_path(out_dir.join("streaming_descriptor.bin")) - .compile_protos(&["../proto/v1/streaming.proto"], &["../proto"])?; - Ok(()) -} diff --git a/subprojects/hydra-queue-runner/examples/collect-fods.rs b/subprojects/hydra-queue-runner/examples/collect-fods.rs index 4443fbc86..86ba228c9 100644 --- a/subprojects/hydra-queue-runner/examples/collect-fods.rs +++ b/subprojects/hydra-queue-runner/examples/collect-fods.rs @@ -1,18 +1,25 @@ +use harmonia_store_derivation::derivation::Derivation; +use harmonia_store_path::StorePath; + #[tokio::main] -async fn main() -> anyhow::Result<()> { - let p = nix_utils::parse_store_path("dzgpbp0vp7lj7lgj26rjgmnjicq2wf4k-hello-2.12.2.drv"); +async fn main() -> color_eyre::eyre::Result<()> { + let p: StorePath = "dzgpbp0vp7lj7lgj26rjgmnjicq2wf4k-hello-2.12.2.drv".parse()?; let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(4); - let fod = std::sync::Arc::new(hydra_queue_runner::state::FodChecker::new(Some(tx))); + let nix_config = + daemon_client_utils::parse_nix_remote().map_err(|e| color_eyre::eyre::eyre!(e))?; + let pool = harmonia_store_remote::ConnectionPool::new( + &nix_config.socket, + harmonia_store_remote::PoolConfig::default(), + ); + let fod = std::sync::Arc::new(hydra_queue_runner::state::FodChecker::new(pool, Some(tx))); fod.clone().start_traverse_loop(); fod.to_traverse(&p); fod.trigger_traverse(); let _ = rx.recv().await; - fod.process( - async move |path: nix_utils::StorePath, _: nix_utils::Derivation| { - println!("{path}"); - }, - ) + fod.process(async move |path: StorePath, _: Derivation| { + println!("{path}"); + }) .await; Ok(()) } diff --git a/subprojects/hydra-queue-runner/package.nix b/subprojects/hydra-queue-runner/package.nix index d45cf3489..2bc413eda 100644 --- a/subprojects/hydra-queue-runner/package.nix +++ b/subprojects/hydra-queue-runner/package.nix @@ -4,7 +4,6 @@ rustPlatform, - nixComponents, protobuf, pkg-config, rust-jemalloc-sys, @@ -23,7 +22,6 @@ rustPlatform.buildRustPackage { ../../.cargo ../../.sqlx ../../subprojects/hydra-queue-runner/Cargo.toml - ../../subprojects/hydra-queue-runner/build.rs ../../subprojects/hydra-queue-runner/src ../../subprojects/hydra-queue-runner/examples ../../subprojects/crates @@ -35,9 +33,7 @@ rustPlatform.buildRustPackage { cargoLock = { lockFile = ../../Cargo.lock; - outputHashes = { - "harmonia-store-core-0.0.0-alpha.0" = "sha256-T6Mbhet2sNGqU9wT5keCAKCSJKrDJ1NuuvtmWp7XUPY="; - }; + outputHashes = import ../../packaging/cargo-output-hashes.nix; }; # The source fileset above intentionally excludes hydra-builder, @@ -56,7 +52,6 @@ rustPlatform.buildRustPackage { ]; buildInputs = [ - nixComponents.nix-main protobuf rust-jemalloc-sys ]; diff --git a/subprojects/hydra-queue-runner/src/config.rs b/subprojects/hydra-queue-runner/src/config.rs index 4130c1634..b41424a6d 100644 --- a/subprojects/hydra-queue-runner/src/config.rs +++ b/subprojects/hydra-queue-runner/src/config.rs @@ -1,8 +1,32 @@ use std::{net::SocketAddr, sync::Arc}; -use anyhow::Context as _; use clap::Parser; +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("missing option: {0}")] + MissingOption(&'static str), + + #[error("environment variable `{0}` is missing")] + MissingEnvVar(&'static str), + + #[error("parsing Nix remote: {0}")] + ParseNixStore(String), + + #[error("reading configuration file")] + Io(#[from] std::io::Error), + + #[error("{context}")] + Toml { + context: String, + #[source] + source: toml::de::Error, + }, + + #[error("preparing configuration")] + Prepare(#[source] Box), +} + #[derive(Debug, Clone)] pub enum BindSocket { Tcp(SocketAddr), @@ -86,20 +110,20 @@ impl Cli { #[tracing::instrument(skip(self), err)] pub async fn get_mtls( &self, - ) -> anyhow::Result<(tonic::transport::Certificate, tonic::transport::Identity)> { + ) -> Result<(tonic::transport::Certificate, tonic::transport::Identity), ConfigError> { let server_cert_path = self .server_cert_path .as_deref() - .ok_or_else(|| anyhow::anyhow!("server_cert_path not provided"))?; + .ok_or(ConfigError::MissingOption("server_cert_path"))?; let server_key_path = self .server_key_path .as_deref() - .ok_or_else(|| anyhow::anyhow!("server_key_path not provided"))?; + .ok_or(ConfigError::MissingOption("server_key_path"))?; let client_ca_cert_path = self .client_ca_cert_path .as_deref() - .ok_or_else(|| anyhow::anyhow!("client_ca_cert_path not provided"))?; + .ok_or(ConfigError::MissingOption("client_ca_cert_path"))?; let client_ca_cert = fs_err::tokio::read_to_string(client_ca_cert_path).await?; let client_ca_cert = tonic::transport::Certificate::from_pem(client_ca_cert); @@ -288,7 +312,7 @@ pub struct PreparedApp { } impl TryFrom for PreparedApp { - type Error = anyhow::Error; + type Error = ConfigError; fn try_from(val: AppConfig) -> Result { let remote_store_addr = val @@ -302,18 +326,17 @@ impl TryFrom for PreparedApp { }) .collect(); - let logname = std::env::var("LOGNAME").context("LOGNAME env var missing")?; - let nix_state_dir = - std::env::var("NIX_STATE_DIR").unwrap_or_else(|_| "/nix/var/nix/".to_owned()); - let roots_dir = val.roots_dir.map_or_else( - || { - std::path::PathBuf::from(nix_state_dir) - .join("gcroots/per-user") - .join(logname) - .join("hydra-roots") - }, - |roots_dir| roots_dir, - ); + let logname = + std::env::var("LOGNAME").map_err(|_| ConfigError::MissingEnvVar("LOGNAME"))?; + let nix_remote = + daemon_client_utils::parse_nix_remote().map_err(ConfigError::ParseNixStore)?; + let roots_dir = val.roots_dir.unwrap_or_else(|| { + nix_remote + .state_dir + .join("gcroots/per-user") + .join(logname) + .join("hydra-roots") + }); fs_err::create_dir_all(&roots_dir)?; let hydra_log_dir = val.hydra_data_dir.join("build-logs"); @@ -388,18 +411,24 @@ impl TryFrom for PreparedApp { /// Loads the config from specified path #[tracing::instrument(err)] -fn load_config(filepath: &str) -> anyhow::Result { +fn load_config(filepath: &str) -> Result { tracing::info!("Trying to loading file: {filepath}"); let toml: AppConfig = if let Ok(content) = fs_err::read_to_string(filepath) { - toml::from_str(&content) - .with_context(|| format!("Failed to toml load from '{filepath}'"))? + toml::from_str(&content).map_err(|source| ConfigError::Toml { + context: format!("loading config from '{filepath}'"), + source, + })? } else { tracing::warn!("no config file found! Using default config"); - toml::from_str("").context("Failed to parse empty string as config")? + toml::from_str("").map_err(|source| ConfigError::Toml { + context: "parsing empty string as config".to_owned(), + source, + })? }; tracing::info!("Loaded config: {toml:?}"); - toml.try_into().context("Failed to prepare configuration") + toml.try_into() + .map_err(|e| ConfigError::Prepare(Box::new(e))) } #[derive(Debug, Clone)] @@ -409,7 +438,7 @@ pub struct App { impl App { #[tracing::instrument(err)] - pub fn init(filepath: &str) -> anyhow::Result { + pub fn init(filepath: &str) -> Result { Ok(Self { inner: Arc::new(arc_swap::ArcSwap::from(Arc::new(load_config(filepath)?))), }) diff --git a/subprojects/hydra-queue-runner/src/io/build.rs b/subprojects/hydra-queue-runner/src/io/build.rs index d5d353440..3905949df 100644 --- a/subprojects/hydra-queue-runner/src/io/build.rs +++ b/subprojects/hydra-queue-runner/src/io/build.rs @@ -1,10 +1,11 @@ +use harmonia_store_path::StorePath; use std::sync::atomic::Ordering; #[derive(Debug, Clone, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct Build { id: db::models::BuildID, - drv_path: nix_utils::StorePath, + drv_path: StorePath, jobset_id: crate::state::JobsetID, name: String, timestamp: jiff::Timestamp, diff --git a/subprojects/hydra-queue-runner/src/io/machine.rs b/subprojects/hydra-queue-runner/src/io/machine.rs index 50e0e7b32..4e6d6f8bb 100644 --- a/subprojects/hydra-queue-runner/src/io/machine.rs +++ b/subprojects/hydra-queue-runner/src/io/machine.rs @@ -1,3 +1,4 @@ +use harmonia_store_path::StorePath; use std::sync::{Arc, atomic::Ordering}; use smallvec::SmallVec; @@ -84,16 +85,18 @@ impl MachineStats { avg_step_import_time_ms, avg_step_build_time_ms, avg_step_upload_time_ms, - ) = if nr_steps_done > 0 { - ( - total_step_time_ms / nr_steps_done, - total_step_import_time_ms / nr_steps_done, - total_step_build_time_ms / nr_steps_done, - total_step_upload_time_ms / nr_steps_done, - ) - } else { - (0, 0, 0, 0) - }; + ) = ( + total_step_time_ms.checked_div(nr_steps_done).unwrap_or(0), + total_step_import_time_ms + .checked_div(nr_steps_done) + .unwrap_or(0), + total_step_build_time_ms + .checked_div(nr_steps_done) + .unwrap_or(0), + total_step_upload_time_ms + .checked_div(nr_steps_done) + .unwrap_or(0), + ); Self { current_jobs: item.get_current_jobs(), @@ -162,7 +165,7 @@ pub struct Machine { use_substitutes: bool, nix_version: String, stats: MachineStats, - jobs: Vec, + jobs: Vec, has_capacity: bool, has_dynamic_capacity: bool, diff --git a/subprojects/hydra-queue-runner/src/io/response_types.rs b/subprojects/hydra-queue-runner/src/io/response_types.rs index ada81c2d7..5d03c7270 100644 --- a/subprojects/hydra-queue-runner/src/io/response_types.rs +++ b/subprojects/hydra-queue-runner/src/io/response_types.rs @@ -1,7 +1,5 @@ use hashbrown::HashMap; -use nix_utils::BaseStore as _; - use super::{ Build, Jobset, Machine, QueueRunnerStats, Step, StepInfo, stats::S3Stats, stats::StoreStats, }; @@ -39,12 +37,9 @@ impl DumpResponse { queue_runner: QueueRunnerStats, machines: HashMap, jobsets: HashMap, - local_store: &nix_utils::LocalStore, remote_stores: &[binary_cache::S3BinaryCacheClient], ) -> Self { - let store = local_store - .get_store_stats() - .map_or(None, |s| Some(StoreStats::new(&s))); + let store: Option = None; // not available via daemon protocol Self { queue_runner, diff --git a/subprojects/hydra-queue-runner/src/io/stats.rs b/subprojects/hydra-queue-runner/src/io/stats.rs index 1dbfb4514..55f73b3f0 100644 --- a/subprojects/hydra-queue-runner/src/io/stats.rs +++ b/subprojects/hydra-queue-runner/src/io/stats.rs @@ -1,4 +1,24 @@ -use anyhow::Context as _; +#[derive(Debug, thiserror::Error)] +pub enum CgroupError { + #[error("reading cgroup file")] + Io(#[from] std::io::Error), + + #[error("reading process info")] + Proc(#[from] procfs::ProcError), + + #[error("failed to parse cgroup value `{field}`")] + Parse { + field: &'static str, + #[source] + source: std::num::ParseIntError, + }, + + #[error("cgroup information is missing in process")] + NoCgroup, + + #[error("cgroups directory does not exist")] + NoCgroupDir, +} #[derive(Debug, Clone, Copy, serde::Serialize)] #[serde(rename_all = "camelCase")] @@ -36,24 +56,36 @@ pub struct MemoryStats { impl MemoryStats { #[tracing::instrument(err)] - fn new(cgroups_path: &std::path::Path) -> anyhow::Result { + fn new(cgroups_path: &std::path::Path) -> Result { Ok(Self { current_bytes: fs_err::read_to_string(cgroups_path.join("memory.current"))? .trim() .parse() - .context("memory current parsing failed")?, + .map_err(|source| CgroupError::Parse { + field: "memory.current", + source, + })?, peak_bytes: fs_err::read_to_string(cgroups_path.join("memory.peak"))? .trim() .parse() - .context("memory peak parsing failed")?, + .map_err(|source| CgroupError::Parse { + field: "memory.peak", + source, + })?, swap_current_bytes: fs_err::read_to_string(cgroups_path.join("memory.swap.current"))? .trim() .parse() - .context("swap parsing failed")?, + .map_err(|source| CgroupError::Parse { + field: "memory.swap.current", + source, + })?, zswap_current_bytes: fs_err::read_to_string(cgroups_path.join("memory.zswap.current"))? .trim() .parse() - .context("zswap parsing failed")?, + .map_err(|source| CgroupError::Parse { + field: "memory.zswap.current", + source, + })?, }) } } @@ -67,7 +99,7 @@ pub struct IoStats { impl IoStats { #[tracing::instrument(err)] - fn new(cgroups_path: &std::path::Path) -> anyhow::Result { + fn new(cgroups_path: &std::path::Path) -> Result { let mut total_read_bytes: u64 = 0; let mut total_write_bytes: u64 = 0; @@ -108,7 +140,7 @@ pub struct CpuStats { impl CpuStats { #[tracing::instrument(err)] - fn new(cgroups_path: &std::path::Path) -> anyhow::Result { + fn new(cgroups_path: &std::path::Path) -> Result { let contents = fs_err::read_to_string(cgroups_path.join("cpu.stat"))?; let mut usage_usec: u128 = 0; @@ -154,18 +186,18 @@ pub struct CgroupStats { impl CgroupStats { #[tracing::instrument(err)] - fn new(me: &procfs::process::Process) -> anyhow::Result { + fn new(me: &procfs::process::Process) -> Result { let cgroups_pathname = format!( "/sys/fs/cgroup/{}", me.cgroups()? .0 .first() - .ok_or_else(|| anyhow::anyhow!("cgroup information is missing in process."))? + .ok_or(CgroupError::NoCgroup)? .pathname ); let cgroups_path = std::path::Path::new(&cgroups_pathname); if !cgroups_path.exists() { - return Err(anyhow::anyhow!("cgroups directory does not exists.")); + return Err(CgroupError::NoCgroupDir); } Ok(Self { @@ -232,7 +264,7 @@ pub struct StoreStats { impl StoreStats { #[must_use] - pub fn new(v: &nix_utils::StoreStats) -> Self { + pub fn new(v: &StoreStats) -> Self { Self { nar_info_read: v.nar_info_read, nar_info_read_averted: v.nar_info_read_averted, @@ -247,8 +279,8 @@ impl StoreStats { nar_write_bytes: v.nar_write_bytes, nar_write_compressed_bytes: v.nar_write_compressed_bytes, nar_write_compression_time_ms: v.nar_write_compression_time_ms, - nar_compression_savings: v.nar_compression_savings(), - nar_compression_speed: v.nar_compression_speed(), + nar_compression_savings: 0.0, // not available via daemon protocol + nar_compression_speed: 0.0, // not available via daemon protocol } } } diff --git a/subprojects/hydra-queue-runner/src/io/step.rs b/subprojects/hydra-queue-runner/src/io/step.rs index 276db54ae..6b649ca16 100644 --- a/subprojects/hydra-queue-runner/src/io/step.rs +++ b/subprojects/hydra-queue-runner/src/io/step.rs @@ -1,10 +1,11 @@ +use harmonia_store_path::StorePath; use std::sync::atomic::Ordering; #[derive(Debug, serde::Serialize)] #[serde(rename_all = "camelCase")] #[allow(clippy::struct_excessive_bools)] pub struct Step { - drv_path: nix_utils::StorePath, + drv_path: StorePath, runnable: bool, finished: bool, previous_failure: bool, diff --git a/subprojects/hydra-queue-runner/src/io/step_info.rs b/subprojects/hydra-queue-runner/src/io/step_info.rs index 8598a9adf..eea4cbc1f 100644 --- a/subprojects/hydra-queue-runner/src/io/step_info.rs +++ b/subprojects/hydra-queue-runner/src/io/step_info.rs @@ -1,10 +1,11 @@ +use harmonia_store_path::StorePath; use std::sync::atomic::Ordering; #[allow(clippy::struct_excessive_bools)] #[derive(Debug, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct StepInfo { - drv_path: nix_utils::StorePath, + drv_path: StorePath, already_scheduled: bool, runnable: bool, finished: bool, diff --git a/subprojects/hydra-queue-runner/src/io/uploads.rs b/subprojects/hydra-queue-runner/src/io/uploads.rs index 38a401502..29f03c6e0 100644 --- a/subprojects/hydra-queue-runner/src/io/uploads.rs +++ b/subprojects/hydra-queue-runner/src/io/uploads.rs @@ -1,13 +1,14 @@ +use harmonia_store_path::StorePath; #[derive(Debug, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct UploadsResponse { - paths: Vec, + paths: Vec, path_count: usize, } impl UploadsResponse { #[must_use] - pub const fn new(paths: Vec) -> Self { + pub const fn new(paths: Vec) -> Self { Self { path_count: paths.len(), paths, diff --git a/subprojects/hydra-queue-runner/src/main.rs b/subprojects/hydra-queue-runner/src/main.rs index ae4a2d449..780e2db06 100644 --- a/subprojects/hydra-queue-runner/src/main.rs +++ b/subprojects/hydra-queue-runner/src/main.rs @@ -23,10 +23,13 @@ pub mod utils; use std::future::Future; -use anyhow::Context as _; +use color_eyre::eyre::{self, WrapErr as _}; use state::State; +type GrpcServer = + std::pin::Pin> + Send>>; + #[cfg(not(target_env = "msvc"))] #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; @@ -75,7 +78,8 @@ fn spawn_config_reloader( } #[tokio::main] -async fn main() -> anyhow::Result<()> { +#[allow(clippy::too_many_lines)] +async fn main() -> color_eyre::Result<()> { let _tracing_guard = hydra_tracing::init()?; #[cfg(debug_assertions)] @@ -89,12 +93,11 @@ async fn main() -> anyhow::Result<()> { })); } - nix_utils::init_nix(); let state = State::new().await?; let lockfile_path = state.config.get_lockfile(); let _lock = lock_file::LockFile::acquire(&lockfile_path) - .context("Another instance is already running.")?; + .wrap_err("Another instance is already running.")?; state.clear_busy().await?; // clear busy once before starting the queue-runner @@ -102,7 +105,7 @@ async fn main() -> anyhow::Result<()> { tracing::error!( "mtls configured inproperly, please pass all options: server_cert_path, server_key_path and client_ca_cert_path!" ); - return Err(anyhow::anyhow!("Configuration issue")); + return Err(eyre::eyre!("Configuration issue")); } let task_abort_handles = start_task_loops(&state); @@ -117,28 +120,22 @@ async fn main() -> anyhow::Result<()> { .collect(); let http_listener = match &state.cli.rest_bind { - config::BindSocket::Tcp(s) => { - let listener = tokio::net::TcpListener::bind(s).await?; - listener - } + config::BindSocket::Tcp(s) => tokio::net::TcpListener::bind(s).await?, config::BindSocket::ListenFd => { let idx = fd_names.iter().position(|n| n == "rest").unwrap_or(0); - let std_listener = listenfd.take_tcp_listener(idx)?.ok_or_else(|| { - anyhow::anyhow!("No listenfd TCP listener at index {idx} for REST") - })?; + let std_listener = listenfd + .take_tcp_listener(idx)? + .ok_or_else(|| eyre::eyre!("No listenfd TCP listener at index {idx} for REST"))?; std_listener.set_nonblocking(true)?; tokio::net::TcpListener::from_std(std_listener)? } config::BindSocket::Unix(_) => { - anyhow::bail!("HTTP server does not support Unix sockets"); + return Err(eyre::eyre!("HTTP server does not support Unix sockets")); } }; let http_addr = http_listener.local_addr()?; - let (srv1, grpc_info): ( - std::pin::Pin> + Send>>, - String, - ) = match &state.cli.grpc_bind { + let (srv1, grpc_info): (GrpcServer, String) = match &state.cli.grpc_bind { config::BindSocket::Tcp(s) => { let listener = tokio::net::TcpListener::bind(s).await?; let addr = listener.local_addr()?; @@ -150,9 +147,9 @@ async fn main() -> anyhow::Result<()> { } config::BindSocket::ListenFd => { let idx = fd_names.iter().position(|n| n == "grpc").unwrap_or(1); - let std_listener = listenfd.take_tcp_listener(idx)?.ok_or_else(|| { - anyhow::anyhow!("No listenfd TCP listener at index {idx} for gRPC") - })?; + let std_listener = listenfd + .take_tcp_listener(idx)? + .ok_or_else(|| eyre::eyre!("No listenfd TCP listener at index {idx} for gRPC"))?; let addr = std_listener.local_addr()?; let info = addr.to_string(); std_listener.set_nonblocking(true)?; @@ -183,9 +180,13 @@ async fn main() -> anyhow::Result<()> { let task = tokio::spawn(async move { match futures_util::future::join(srv1, srv2).await { (Ok(()), Ok(())) => Ok(()), - (Ok(()), Err(e)) => Err(anyhow::anyhow!("hyper error while awaiting handle: {e}")), - (Err(e), Ok(())) => Err(anyhow::anyhow!("tonic error while awaiting handle: {e}")), - (Err(e1), Err(e2)) => Err(anyhow::anyhow!( + (Ok(()), Err(e)) => Err(color_eyre::eyre::eyre!( + "hyper error while awaiting handle: {e}" + )), + (Err(e), Ok(())) => Err(color_eyre::eyre::eyre!( + "tonic error while awaiting handle: {e}" + )), + (Err(e1), Err(e2)) => Err(color_eyre::eyre::eyre!( "tonic and hyper error while awaiting handle: {e1} | {e2}" )), } diff --git a/subprojects/hydra-queue-runner/src/server/grpc.rs b/subprojects/hydra-queue-runner/src/server/grpc.rs index 41dc97b15..e318543a4 100644 --- a/subprojects/hydra-queue-runner/src/server/grpc.rs +++ b/subprojects/hydra-queue-runner/src/server/grpc.rs @@ -1,111 +1,42 @@ use std::sync::Arc; -use anyhow::Context as _; +use harmonia_store_path::StorePath; +/// Errors from gRPC server setup and serving. +#[derive(Debug, thiserror::Error)] +pub enum ServerError { + #[error("gRPC transport error")] + Transport(#[from] tonic::transport::Error), + #[error("loading mTLS certificate and identity")] + Mtls(#[from] crate::config::ConfigError), + #[error("reflection service: {0}")] + Reflection(#[from] tonic_reflection::server::Error), +} use tokio::sync::mpsc; use tonic::service::interceptor::InterceptedService; use tower::ServiceBuilder; use tracing::Instrument as _; -use crate::{ - server::grpc::runner_v1::{BuildResultState, StepUpdate}, - state::{Machine, MachineMessage, State}, -}; -use nix_utils::BaseStore as _; - -include!(concat!(env!("OUT_DIR"), "/proto_version.rs")); -use runner_v1::runner_service_server::{RunnerService, RunnerServiceServer}; -use runner_v1::{ - BuildResultInfo, BuilderRequest, FetchRequisitesRequest, JoinResponse, LogChunk, NarData, +use crate::state::{Machine, MachineMessage, State}; +use hydra_proto::ProtoStorePath; +use hydra_proto::{ + BuildResultInfo, BuilderRequest, JoinResponse, LogChunk, PROTO_API_VERSION, PresignedUploadComplete, PresignedUrlRequest, PresignedUrlResponse, RunnerRequest, - SimplePingMessage, StorePaths, VersionCheckRequest, VersionCheckResponse, builder_request, + SimplePingMessage, StepUpdate, VersionCheckRequest, VersionCheckResponse, builder_request, + runner_service_server::{RunnerService, RunnerServiceServer}, }; -use shared::proto::ProtoStorePath; type BuilderResult = Result, tonic::Status>; type OpenTunnelResponseStream = std::pin::Pin> + Send>>; -type StreamFileResponseStream = - std::pin::Pin> + Send>>; - -type CompressionEncoder = async_compression::tokio::bufread::ZstdEncoder; +type FetchPathsResponseStream = std::pin::Pin< + Box> + Send>, +>; type CompressionDecoder = async_compression::tokio::bufread::ZstdDecoder; -const DUPLEX_BUFFER_SIZE: usize = 256 * 1024; - -fn decode_stream( - stream: S, -) -> impl tokio_stream::Stream> -where - S: tokio_stream::Stream> + Unpin, -{ - let reader = tokio_util::io::StreamReader::new(stream); - let decoder = CompressionDecoder::new(reader); - tokio_util::io::ReaderStream::new(decoder) -} - -fn compression_encode_channel() -> ( - tokio::io::DuplexStream, - mpsc::UnboundedReceiver>, -) { - let (raw_writer, raw_reader) = tokio::io::duplex(DUPLEX_BUFFER_SIZE); - let (tx, rx) = mpsc::unbounded_channel(); - - tokio::task::spawn(async move { - use tokio_stream::StreamExt as _; - let encoder = CompressionEncoder::new(tokio::io::BufReader::new(raw_reader)); - let mut stream = tokio_util::io::ReaderStream::new(encoder); - while let Some(chunk) = stream.next().await { - match chunk { - Ok(bytes) => { - if tx - .send(Ok(NarData { - chunk: bytes.into(), - })) - .is_err() - { - break; - } - } - Err(e) => { - tracing::error!("Failed to compress chunk: {e}"); - let _ = tx.send(Err(tonic::Status::internal("Compression error"))); - break; - } - } - } - }); - - (raw_writer, rx) -} - // there is no reason to make this configurable, it only exists so we ensure the channel is not // closed. we dont use this to write any actual information. const BACKWARDS_PING_INTERVAL: u64 = 30; -pub mod runner_v1 { - // We need to allow pedantic here because of generated code - #![allow(clippy::pedantic, unused_qualifications)] - - tonic::include_proto!("runner.v1"); - - pub(crate) const FILE_DESCRIPTOR_SET: &[u8] = - tonic::include_file_descriptor_set!("streaming_descriptor"); - - impl From for db::models::StepStatus { - fn from(item: StepStatus) -> Self { - match item { - StepStatus::Preparing => Self::Preparing, - StepStatus::Connecting => Self::Connecting, - StepStatus::SeningInputs => Self::SendingInputs, - StepStatus::Building => Self::Building, - StepStatus::WaitingForLocalSlot => Self::WaitingForLocalSlot, - StepStatus::ReceivingOutputs => Self::ReceivingOutputs, - StepStatus::PostProcessing => Self::PostProcessing, - } - } - } -} - fn match_for_io_error(err_status: &tonic::Status) -> Option<&std::io::Error> { let mut err: &(dyn std::error::Error + 'static) = err_status; @@ -179,7 +110,10 @@ pub struct Server { impl Server { /// Serve on a pre-bound TCP listener. #[tracing::instrument(skip(listener, state), err)] - pub async fn run(listener: tokio::net::TcpListener, state: Arc) -> anyhow::Result<()> { + pub async fn run( + listener: tokio::net::TcpListener, + state: Arc, + ) -> Result<(), ServerError> { let stream = tokio_stream::wrappers::TcpListenerStream::new(listener); Self::serve_incoming(stream, state).await } @@ -189,12 +123,12 @@ impl Server { pub async fn run_unix( listener: tokio::net::UnixListener, state: Arc, - ) -> anyhow::Result<()> { + ) -> Result<(), ServerError> { let stream = tokio_stream::wrappers::UnixListenerStream::new(listener); Self::serve_incoming(stream, state).await } - async fn serve_incoming(incoming: S, state: Arc) -> anyhow::Result<()> + async fn serve_incoming(incoming: S, state: Arc) -> Result<(), ServerError> where S: futures_util::Stream>, IO: tokio::io::AsyncRead @@ -203,6 +137,8 @@ impl Server { + Unpin + Send + 'static, + // Required by tonic's `serve_with_incoming` API. We cannot replace this bound with + // something else. IE: Into>, { let service = RunnerServiceServer::new(Self { @@ -231,11 +167,7 @@ impl Server { if state.cli.mtls_enabled() { tracing::info!("Using mtls"); - let (client_ca_cert, server_identity) = state - .cli - .get_mtls() - .await - .context("Failed to get_mtls Certificate and Identity")?; + let (client_ca_cert, server_identity) = state.cli.get_mtls().await?; let tls = tonic::transport::ServerTlsConfig::new() .identity(server_identity) @@ -243,7 +175,7 @@ impl Server { server = server.tls_config(tls)?; } let reflection_service = tonic_reflection::server::Builder::configure() - .register_encoded_file_descriptor_set(runner_v1::FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(hydra_proto::FILE_DESCRIPTOR_SET) .build_v1()?; let (_health_reporter, health_service) = tonic_health::server::health_reporter(); @@ -261,8 +193,7 @@ impl Server { #[tonic::async_trait] impl RunnerService for Server { type OpenTunnelStream = OpenTunnelResponseStream; - type StreamFileStream = StreamFileResponseStream; - type StreamFilesStream = StreamFileResponseStream; + type FetchPathsStream = FetchPathsResponseStream; #[tracing::instrument(skip(self, req), err)] async fn check_version( @@ -286,7 +217,7 @@ impl RunnerService for Server { })); } - let our_store_dir = self.state.store.store_dir().to_string(); + let our_store_dir = self.state.pool.store_dir().to_string(); if req.store_dir != our_store_dir { return Err(tonic::Status::failed_precondition(format!( "Store dir mismatch: builder has `{}`, server has `{}`", @@ -348,7 +279,7 @@ impl RunnerService for Server { let (output_tx, output_rx) = mpsc::channel(128); if let Err(e) = output_tx .send(Ok(RunnerRequest { - message: Some(runner_v1::runner_request::Message::Join(JoinResponse { + message: Some(hydra_proto::runner_request::Message::Join(JoinResponse { machine_id: machine_id.to_string(), max_concurrent_downloads: state.config.get_max_concurrent_downloads(), })), @@ -366,7 +297,7 @@ impl RunnerService for Server { tokio::select! { _ = ping_interval.tick() => { let msg = RunnerRequest { - message: Some(runner_v1::runner_request::Message::Ping(SimplePingMessage { + message: Some(hydra_proto::runner_request::Message::Ping(SimplePingMessage { message: "ping".into(), })) }; @@ -426,7 +357,7 @@ impl RunnerService for Server { async fn build_log( &self, req: tonic::Request>, - ) -> BuilderResult { + ) -> BuilderResult { use tokio_stream::StreamExt as _; let stream = req.into_inner(); @@ -463,37 +394,31 @@ impl RunnerService for Server { .await .map_err(|e| tonic::Status::internal(format!("Failed to write log file: {e}")))?; - Ok(tonic::Response::new(runner_v1::Empty {})) + Ok(tonic::Response::new(hydra_proto::Empty {})) } #[tracing::instrument(skip(self, req), err)] async fn build_result( &self, - req: tonic::Request>, - ) -> BuilderResult { - let stream = req.into_inner(); - - // We leak memory if we use the store from state, so we open and close a new - // connection for each import. This sucks but using the state.store will result in the path - // not being closed! - { - let data_stream = tokio_stream::StreamExt::map(stream, |s| { - s.map(|v| bytes::Bytes::from(v.chunk)) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::UnexpectedEof, e)) - }); - - let store = nix_utils::LocalStore::init(); - store.import_paths(decode_stream(data_stream), false).await - } - .map_err(|_| tonic::Status::internal("Failed to import path."))?; - Ok(tonic::Response::new(runner_v1::Empty {})) + req: tonic::Request>, + ) -> BuilderResult { + let mut guard = self + .state + .pool + .acquire() + .await + .map_err(|e| tonic::Status::internal(format!("daemon connection failed: {e}")))?; + store_transfer::import::import(&mut guard, req.into_inner()) + .await + .map_err(|e| tonic::Status::internal(format!("import error: {e}")))?; + Ok(tonic::Response::new(hydra_proto::Empty {})) } #[tracing::instrument(skip(self), err)] async fn build_step_update( &self, req: tonic::Request, - ) -> BuilderResult { + ) -> BuilderResult { let state = self.state.clone(); let req = req.into_inner(); @@ -524,14 +449,14 @@ impl RunnerService for Server { }.in_current_span() }); - Ok(tonic::Response::new(runner_v1::Empty {})) + Ok(tonic::Response::new(hydra_proto::Empty {})) } #[tracing::instrument(skip(self, req), fields(machine_id=req.get_ref().machine_id, build_id=req.get_ref().build_id), err)] async fn complete_build( &self, req: tonic::Request, - ) -> BuilderResult { + ) -> BuilderResult { let state = self.state.clone(); let req = req.into_inner(); @@ -546,15 +471,14 @@ impl RunnerService for Server { tokio::spawn({ async move { - if req.result_state() == BuildResultState::Success { - let build_output = - match crate::state::BuildOutput::from_grpc(state.store.store_dir(), req) { - Ok(output) => output, - Err(e) => { - tracing::error!("Failed to parse build output: {e}"); - return; - } - }; + if req.result_state() == hydra_proto::BuildResultState::Success { + let build_output = match crate::state::BuildOutput::from_grpc(req) { + Ok(output) => output, + Err(e) => { + tracing::error!("Failed to parse build output: {e}"); + return; + } + }; if let Err(e) = state .succeed_step_by_uuid(build_id, machine_id, build_output) .await @@ -582,34 +506,29 @@ impl RunnerService for Server { .in_current_span() }); - Ok(tonic::Response::new(runner_v1::Empty {})) + Ok(tonic::Response::new(hydra_proto::Empty {})) } #[tracing::instrument(skip(self, req), err)] - async fn fetch_drv_requisites( + async fn fetch_requisites( &self, - req: tonic::Request, - ) -> BuilderResult { + req: tonic::Request, + ) -> BuilderResult { let state = self.state.clone(); - let req = req.into_inner(); - let drv = req - .path - .ok_or_else(|| tonic::Status::invalid_argument("missing path"))? - .0; - - let requisites = state - .store - .query_requisites(&[&drv], req.include_outputs) - .await - .map_err(|e| { - tracing::error!("failed to toposort drv e={e}"); - tonic::Status::internal("failed to toposort drv.") - })? - .into_iter() - .map(ProtoStorePath::from) - .collect(); - - Ok(tonic::Response::new(runner_v1::DrvRequisitesMessage { + let paths: Vec<_> = req.into_inner().paths.into_iter().map(|p| p.0).collect(); + + let requisites: Vec = + daemon_client_utils::query_closure(&state.pool, &paths) + .await + .map_err(|e| { + tracing::error!("failed to compute closure e={e}"); + tonic::Status::internal("failed to compute closure.") + })? + .into_iter() + .map(|vpi| ProtoStorePath(vpi.path)) + .collect(); + + Ok(tonic::Response::new(hydra_proto::RequisitesResponse { requisites, })) } @@ -618,62 +537,57 @@ impl RunnerService for Server { async fn has_path( &self, req: tonic::Request, - ) -> BuilderResult { + ) -> BuilderResult { let path = req.into_inner().0; let state = self.state.clone(); - let has_path = state.store.is_valid_path(&path).await; + let has_path: bool = daemon_client_utils::is_valid_path(&state.pool, &path) + .await + .map_err(|e| tonic::Status::internal(format!("is_valid_path failed: {e}")))?; - Ok(tonic::Response::new(runner_v1::HasPathResponse { + Ok(tonic::Response::new(hydra_proto::HasPathResponse { has_path, })) } #[tracing::instrument(skip(self, req), err)] - async fn stream_file( + async fn fetch_paths( &self, - req: tonic::Request, - ) -> BuilderResult { - let path = req.into_inner().0; - let (raw_writer, rx) = compression_encode_channel(); - - tokio::task::spawn_blocking(move || { - let store = nix_utils::LocalStore::init(); - let mut sync_writer = tokio_util::io::SyncIoBridge::new(raw_writer); - let closure = move |data: &[u8]| { - use std::io::Write; - sync_writer.write_all(data).is_ok() - }; - let _ = store.export_paths(&[path], closure); - }); + req: tonic::Request, + ) -> BuilderResult { + let paths: Vec<_> = req.into_inner().paths.into_iter().map(|p| p.0).collect(); - Ok(tonic::Response::new( - Box::pin(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)) - as Self::StreamFileStream, - )) - } + let mut infos = hashbrown::HashMap::with_capacity(paths.len()); + for path in &paths { + let info = daemon_client_utils::query_path_info(&self.state.pool, path) + .await + .map_err(|e| tonic::Status::internal(format!("query_path_info failed: {e}")))? + .ok_or_else(|| tonic::Status::not_found(format!("path '{path}' is not valid")))?; + infos.insert(path.clone(), info); + } - #[tracing::instrument(skip(self, req), err)] - async fn stream_files( - &self, - req: tonic::Request, - ) -> BuilderResult { - let req = req.into_inner(); - let paths = req.paths.into_iter().map(|p| p.0).collect::>(); - let (raw_writer, rx) = compression_encode_channel(); - - tokio::task::spawn_blocking(move || { - let store = nix_utils::LocalStore::init(); - let mut sync_writer = tokio_util::io::SyncIoBridge::new(raw_writer); - let closure = move |data: &[u8]| { - use std::io::Write; - sync_writer.write_all(data).is_ok() - }; - let _ = store.export_paths(&paths, closure); + let (tx, rx) = mpsc::unbounded_channel(); + + tokio::spawn({ + let pool = self.state.pool.clone(); + async move { + let mut guard = match pool.acquire().await { + Ok(g) => g, + Err(e) => { + tracing::error!("export failed: daemon connection: {e}"); + return; + } + }; + if let Err(e) = + store_transfer::export::export(&mut guard, &paths, &infos, &tx).await + { + tracing::error!("export failed: {e}"); + } + } }); Ok(tonic::Response::new( Box::pin(tokio_stream::wrappers::UnboundedReceiverStream::new(rx)) - as Self::StreamFilesStream, + as Self::FetchPathsStream, )) } @@ -707,19 +621,29 @@ impl RunnerService for Server { .iter() .find_map(|s| match s { crate::state::RemoteStoreBackend::S3(s) => Some(s.clone()), - _ => None, + crate::state::RemoteStoreBackend::NixCopy(_) => None, }) .ok_or_else(|| tonic::Status::failed_precondition("No remote store configured"))? }; let mut responses = Vec::new(); for presigned_request in req.request { - let store_path = nix_utils::parse_store_path(&presigned_request.store_path); - + let store_path = StorePath::from_base_path(&presigned_request.store_path) + .map_err(|e| tonic::Status::invalid_argument(format!("bad store path: {e}")))?; + + let proto_hash = presigned_request + .nar_hash + .ok_or_else(|| tonic::Status::invalid_argument("missing nar_hash"))?; + let hash: harmonia_utils_hash::Hash = proto_hash + .try_into() + .map_err(|e: &str| tonic::Status::invalid_argument(e))?; + let nar_hash: harmonia_store_path_info::NarHash = hash + .try_into() + .map_err(|_| tonic::Status::invalid_argument("nar_hash is not sha256"))?; let presigned_response = remote_store .generate_nar_upload_presigned_url( &store_path, - &presigned_request.nar_hash, + &nar_hash, presigned_request.debug_info_build_ids, ) .await @@ -728,10 +652,10 @@ impl RunnerService for Server { tonic::Status::internal("Failed to generate presigned URL") })?; - responses.push(runner_v1::PresignedNarResponse { - store_path: store_path.to_string().clone(), + responses.push(hydra_proto::PresignedNarResponse { + store_path: store_path.to_string(), nar_url: presigned_response.nar_url, - nar_upload: Some(runner_v1::PresignedUpload { + nar_upload: Some(hydra_proto::PresignedUpload { compression_level: presigned_response.nar_upload.get_compression_level_as_i32(), url: presigned_response.nar_upload.url, path: presigned_response.nar_upload.path, @@ -743,7 +667,7 @@ impl RunnerService for Server { }), ls_upload: presigned_response .ls_upload - .map(|ls| runner_v1::PresignedUpload { + .map(|ls| hydra_proto::PresignedUpload { compression_level: ls.get_compression_level_as_i32(), url: ls.url, path: ls.path, @@ -752,7 +676,7 @@ impl RunnerService for Server { debug_info_upload: presigned_response .debug_info_upload .into_iter() - .map(|p| runner_v1::PresignedUpload { + .map(|p| hydra_proto::PresignedUpload { compression_level: p.get_compression_level_as_i32(), url: p.url, path: p.path, @@ -773,14 +697,13 @@ impl RunnerService for Server { fields( build_id=req.get_ref().build_id, machine_id=req.get_ref().machine_id, - store_path=req.get_ref().store_path ), err, )] async fn notify_presigned_upload_complete( &self, req: tonic::Request, - ) -> BuilderResult { + ) -> BuilderResult { let state = self.state.clone(); let req = req.into_inner(); @@ -807,34 +730,42 @@ impl RunnerService for Server { .iter() .find_map(|s| match s { crate::state::RemoteStoreBackend::S3(s) => Some(s.clone()), - _ => None, + crate::state::RemoteStoreBackend::NixCopy(_) => None, }) .ok_or_else(|| tonic::Status::failed_precondition("No remote store configured"))? }; - let narinfo = binary_cache::NarInfo { - store_path: nix_utils::parse_store_path(&req.store_path), - url: req.url.clone(), - compression: remote_store.cfg.compression, - file_hash: binary_cache::parse_hash(&req.file_hash), - file_size: Some(req.file_size), - nar_hash: binary_cache::parse_hash(&req.nar_hash).ok_or_else(|| { - tonic::Status::invalid_argument(format!("invalid nar hash: {}", req.nar_hash)) - })?, - nar_size: req.nar_size, - references: req - .references - .into_iter() - .map(|p| nix_utils::parse_store_path(&p)) - .collect(), - deriver: req.deriver.map(|p| nix_utils::parse_store_path(&p)), - ca: req.ca, - sigs: vec![], - }; - let store_path = narinfo.store_path.clone(); + let proto_nar_info = req + .nar_info + .ok_or_else(|| tonic::Status::invalid_argument("missing nar_info"))?; + + let mut narinfo: binary_cache::NarInfo = proto_nar_info + .try_into() + .map_err(|e: hydra_proto::NarInfoConvertError| tonic::Status::invalid_argument(e.0))?; + + if &narinfo.info.info.store_dir != self.state.pool.store_dir() { + return Err(tonic::Status::invalid_argument(format!( + "store_dir mismatch: expected {}, got {}", + self.state.pool.store_dir(), + narinfo.info.info.store_dir + ))); + } + + // The cache signs and formats narinfos with its own configured store + // dir; for those signatures to match the uploaded paths it must agree + // with the local store (the narinfo was just checked against it above). + debug_assert_eq!(&remote_store.cfg.store_dir, self.state.pool.store_dir()); + + let store_path = narinfo.path.clone(); + + // Override compression from server config + narinfo.info.compression = Some(remote_store.cfg.compression.as_str().to_owned()); + + let url = narinfo.info.url.clone(); + let size = narinfo.info.download_size; let narinfo_url = remote_store - .upload_narinfo_after_presigned_upload(&self.state.store, narinfo) + .upload_narinfo_after_presigned_upload(narinfo) .await .map_err(|e| { tracing::error!("Failed to upload narinfo for {}: {e}", store_path); @@ -842,13 +773,13 @@ impl RunnerService for Server { })?; tracing::debug!( - "Presigned upload completed and narinfo uploaded for path: {}, url: {}, size: {} bytes, narinfo: {}", + "Presigned upload completed and narinfo uploaded for path: {}, url: {:?}, size: {:?} bytes, narinfo: {}", store_path, - req.url, - req.file_size, + url, + size, narinfo_url ); - Ok(tonic::Response::new(runner_v1::Empty {})) + Ok(tonic::Response::new(hydra_proto::Empty {})) } } diff --git a/subprojects/hydra-queue-runner/src/server/http.rs b/subprojects/hydra-queue-runner/src/server/http.rs index 84d123ece..68a3f61f4 100644 --- a/subprojects/hydra-queue-runner/src/server/http.rs +++ b/subprojects/hydra-queue-runner/src/server/http.rs @@ -22,12 +22,18 @@ pub enum Error { #[error("std io error: `{0}`")] Io(#[from] std::io::Error), - #[error("anyhow error: `{0}`")] - Anyhow(#[from] anyhow::Error), + #[error("prometheus error: `{0}`")] + Prometheus(#[from] prometheus::Error), + + #[error(transparent)] + State(#[from] crate::state::StateError), #[error("db error: `{0}`")] Sqlx(#[from] db::Error), + #[error("invalid store path")] + StorePath(#[from] harmonia_store_path::ParseStorePathError), + #[error("Not found")] NotFound, @@ -44,8 +50,10 @@ impl Error { | Self::HyperHttp(_) | Self::Hyper(_) | Self::Io(_) - | Self::Anyhow(_) + | Self::Prometheus(_) + | Self::State(_) | Self::Sqlx(_) + | Self::StorePath(_) | Self::Fatal => hyper::StatusCode::INTERNAL_SERVER_ERROR, Self::NotFound => hyper::StatusCode::NOT_FOUND, } @@ -208,7 +216,7 @@ mod handler { .iter() .filter_map(|s| match s { crate::state::RemoteStoreBackend::S3(s) => Some(s.clone()), - _ => None, + crate::state::RemoteStoreBackend::NixCopy(_) => None, }) .collect() }; @@ -216,7 +224,6 @@ mod handler { queue_stats, machines, jobsets, - &state.store, &s3_stores, )) } @@ -344,6 +351,7 @@ mod handler { use super::super::{Error, Response, construct_json_ok_response}; use crate::{io, state::State}; + use harmonia_store_path::StorePath; #[tracing::instrument(skip(req, state), err)] pub(crate) async fn put( @@ -354,7 +362,7 @@ mod handler { let data: io::BuildPayload = serde_json::from_reader(whole_body.reader())?; state - .queue_one_build(data.jobset_id, &nix_utils::parse_store_path(&data.drv)) + .queue_one_build(data.jobset_id, &StorePath::from_base_path(&data.drv)?) .await?; construct_json_ok_response(&io::Empty {}) } diff --git a/subprojects/hydra-queue-runner/src/state/build.rs b/subprojects/hydra-queue-runner/src/state/build.rs index 0f13211ad..f0a688a61 100644 --- a/subprojects/hydra-queue-runner/src/state/build.rs +++ b/subprojects/hydra-queue-runner/src/state/build.rs @@ -3,20 +3,20 @@ use std::hash::Hash; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; -use anyhow::Context; use hashbrown::{HashMap, HashSet}; use super::{Jobset, JobsetID, Step}; use db::models::{BuildID, BuildStatus}; -use nix_utils::BaseStore as _; +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_path::StorePath; pub(super) type AtomicBuildID = AtomicI32; #[derive(Debug)] pub struct Build { pub id: BuildID, - pub drv_path: nix_utils::StorePath, - pub outputs: BTreeMap, + pub drv_path: StorePath, + pub outputs: BTreeMap, pub jobset_id: JobsetID, pub name: String, pub timestamp: jiff::Timestamp, @@ -49,7 +49,7 @@ impl Hash for Build { impl Build { #[must_use] - pub fn new_debug(drv_path: &nix_utils::StorePath) -> Arc { + pub fn new_debug(drv_path: &StorePath) -> Arc { Arc::new(Self { id: BuildID::MAX, drv_path: drv_path.to_owned(), @@ -68,7 +68,7 @@ impl Build { } #[tracing::instrument(skip(v, jobset), err)] - pub fn new(v: db::models::Build, jobset: Arc) -> anyhow::Result> { + pub fn new(v: db::models::Build, jobset: Arc) -> Result, jiff::Error> { Ok(Arc::new(Self { id: v.id, drv_path: v.drvpath, @@ -156,30 +156,15 @@ impl Build { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BuildResultState { - Success, - BuildFailure, - PreparingFailure, - ImportFailure, - UploadFailure, - PostProcessingFailure, + /// Result reported by the builder over gRPC. + Completed(hydra_proto::BuildResultState), Aborted, Cancelled, } -impl From for BuildResultState { - fn from(v: crate::server::grpc::runner_v1::BuildResultState) -> Self { - match v { - crate::server::grpc::runner_v1::BuildResultState::BuildFailure => Self::BuildFailure, - crate::server::grpc::runner_v1::BuildResultState::Success => Self::Success, - crate::server::grpc::runner_v1::BuildResultState::PreparingFailure => { - Self::PreparingFailure - } - crate::server::grpc::runner_v1::BuildResultState::ImportFailure => Self::ImportFailure, - crate::server::grpc::runner_v1::BuildResultState::UploadFailure => Self::UploadFailure, - crate::server::grpc::runner_v1::BuildResultState::PostProcessingFailure => { - Self::PostProcessingFailure - } - } +impl From for BuildResultState { + fn from(v: hydra_proto::BuildResultState) -> Self { + Self::Completed(v) } } @@ -203,7 +188,7 @@ pub struct RemoteBuild { stop_time: Option, overhead: i32, - pub log_file: String, + pub log_file: std::path::PathBuf, } impl Default for RemoteBuild { @@ -226,7 +211,7 @@ impl RemoteBuild { start_time: None, stop_time: None, overhead: 0, - log_file: String::new(), + log_file: std::path::PathBuf::new(), } } @@ -243,18 +228,24 @@ impl RemoteBuild { } } - pub const fn update_with_result_state(&mut self, state: &BuildResultState) { + const fn update_with_completed(&mut self, state: hydra_proto::BuildResultState) { match state { - BuildResultState::BuildFailure => { + hydra_proto::BuildResultState::BuildFailure => { self.can_retry = false; } - BuildResultState::Success => (), - BuildResultState::PreparingFailure - | BuildResultState::ImportFailure - | BuildResultState::UploadFailure - | BuildResultState::PostProcessingFailure => { + hydra_proto::BuildResultState::Success => (), + hydra_proto::BuildResultState::PreparingFailure + | hydra_proto::BuildResultState::ImportFailure + | hydra_proto::BuildResultState::UploadFailure + | hydra_proto::BuildResultState::PostProcessingFailure => { self.can_retry = true; } + } + } + + pub const fn update_with_result_state(&mut self, state: BuildResultState) { + match state { + BuildResultState::Completed(s) => self.update_with_completed(s), BuildResultState::Aborted => { self.can_retry = true; self.step_status = BuildStatus::Aborted; @@ -340,124 +331,7 @@ impl RemoteBuild { } } -/// Store path with an optional relative path from the store object directory. -/// -/// Build products can reference files inside store outputs (e.g. `/nix/store/hash-name/bin/foo`), -/// so we separate the base `StorePath` from the trailing sub-path. -#[derive(Debug, Clone)] -pub struct RelativeStorePath { - pub base_path: nix_utils::StorePath, - pub relative_path: Box, -} - -impl RelativeStorePath { - pub fn from_path(store_dir: &nix_utils::StoreDir, path: &str) -> anyhow::Result { - let stripped = store_dir - .strip_prefix(path) - .with_context(|| format!("stripping store dir from '{path}'"))?; - let (base_str, remaining_str) = stripped.split_once('/').unwrap_or((stripped, "")); - Ok(Self { - base_path: nix_utils::StorePath::from_base_path(base_str) - .with_context(|| format!("parsing store path from '{base_str}'"))?, - relative_path: remaining_str.into(), - }) - } - - pub fn print(&self, store_dir: &nix_utils::StoreDir) -> String { - if self.relative_path.is_empty() { - store_dir.display(&self.base_path).to_string() - } else { - format!( - "{}/{}", - store_dir.display(&self.base_path), - self.relative_path - ) - } - } -} - -#[derive(Debug, Clone)] -pub struct BuildProduct { - pub path: Option, - pub default_path: Option, - - pub r#type: String, - pub subtype: String, - pub name: String, - - pub is_regular: bool, - - pub sha256hash: Option, - pub file_size: Option, -} - -impl BuildProduct { - pub fn from_db(v: db::models::OwnedBuildProduct) -> Self { - Self { - path: v.path.map(|p| RelativeStorePath { - base_path: p, - relative_path: "".into(), - }), - default_path: v.defaultpath, - r#type: v.r#type, - subtype: v.subtype, - name: v.name, - is_regular: v.filesize.is_some(), - sha256hash: v.sha256hash, - #[allow(clippy::cast_sign_loss)] - file_size: v.filesize.map(|v| v as u64), - } - } - - pub fn from_grpc( - store_dir: &nix_utils::StoreDir, - v: crate::server::grpc::runner_v1::BuildProduct, - ) -> anyhow::Result { - Ok(Self { - path: Some(RelativeStorePath::from_path(store_dir, &v.path)?), - default_path: Some(v.default_path), - r#type: v.r#type, - subtype: v.subtype, - name: v.name, - is_regular: v.is_regular, - sha256hash: v.sha256hash, - file_size: v.file_size, - }) - } - - pub fn from_shared( - store_dir: &nix_utils::StoreDir, - v: shared::BuildProduct, - ) -> anyhow::Result { - Ok(Self { - path: Some(RelativeStorePath::from_path(store_dir, &v.path)?), - default_path: Some(v.default_path), - r#type: v.r#type, - subtype: v.subtype, - name: v.name, - is_regular: v.is_regular, - sha256hash: v.sha256hash, - file_size: v.file_size, - }) - } -} - -#[derive(Debug, Clone)] -pub struct BuildMetric { - pub name: String, - pub unit: Option, - pub value: f64, -} - -impl From for BuildMetric { - fn from(v: db::models::OwnedBuildMetric) -> Self { - Self { - name: v.name, - unit: v.unit, - value: v.value, - } - } -} +pub(crate) use nix_support::{BuildMetric, BuildProduct}; #[derive(Debug, Default, Clone, Copy)] pub struct BuildTimings { @@ -492,19 +366,40 @@ pub struct BuildOutput { pub size: u64, pub products: Vec, - pub outputs: BTreeMap, - pub metrics: Vec, + pub outputs: BTreeMap, + pub metrics: BTreeMap, +} + +/// Everything that can go wrong constructing a [`BuildOutput`], whether +/// parsing an incoming `BuildResultInfo`/db row or reading from the store. +#[derive(Debug, thiserror::Error)] +pub enum BuildOutputError { + #[error("buildstatus missing")] + BuildStatusMissing, + + #[error("buildstatus value did not map to a known status")] + BuildStatusUnknown, + + #[error("output missing path")] + OutputMissingPath, + + #[error("invalid output name")] + OutputName(#[from] harmonia_store_path::StorePathNameError), + + #[error("nix daemon error")] + Daemon(#[from] harmonia_store_remote::DaemonError), + + #[error("reading build output")] + Io(#[from] std::io::Error), } impl TryFrom for BuildOutput { - type Error = anyhow::Error; + type Error = BuildOutputError; - fn try_from(v: db::models::BuildOutput) -> anyhow::Result { - let build_status = BuildStatus::from_i32( - v.buildstatus - .ok_or_else(|| anyhow::anyhow!("buildstatus missing"))?, - ) - .ok_or_else(|| anyhow::anyhow!("buildstatus did not map"))?; + fn try_from(v: db::models::BuildOutput) -> Result { + let build_status = + BuildStatus::from_i32(v.buildstatus.ok_or(BuildOutputError::BuildStatusMissing)?) + .ok_or(BuildOutputError::BuildStatusUnknown)?; Ok(Self { failed: build_status != BuildStatus::Success, timings: BuildTimings::default(), @@ -515,86 +410,74 @@ impl TryFrom for BuildOutput { size: v.size.unwrap_or_default() as u64, products: vec![], outputs: BTreeMap::new(), - metrics: Vec::with_capacity(10), + metrics: BTreeMap::new(), }) } } impl BuildOutput { - pub fn from_grpc( - store_dir: &nix_utils::StoreDir, - v: crate::server::grpc::runner_v1::BuildResultInfo, - ) -> anyhow::Result { + pub fn from_grpc(v: hydra_proto::BuildResultInfo) -> Result { let mut outputs = BTreeMap::new(); let mut closure_size = 0; let mut nar_size = 0; - - for o in v.outputs { - match o.output { - Some(crate::server::grpc::runner_v1::output::Output::Nameonly(_)) => { - // We dont care about outputs that dont have a path, - } - Some(crate::server::grpc::runner_v1::output::Output::Withpath(o)) => { - let path = o - .path - .ok_or_else(|| anyhow::anyhow!("output missing path"))? - .0; - outputs.insert(o.name.parse()?, path); - closure_size += o.closure_size; - nar_size += o.nar_size; - } - None => (), + let mut merged = nix_support::NixSupport::default(); + + for (name, info) in v.output_infos { + let path = info.path.ok_or(BuildOutputError::OutputMissingPath)?.0; + closure_size += info.closure_size; + nar_size += info.nar_size; + outputs.insert(name.parse()?, path); + if let Some(ns) = info.nix_support { + let ns: nix_support::NixSupport = ns.into(); + merged.combine(ns); } } - let (failed, release_name, products, metrics) = if let Some(nix_support) = v.nix_support { - ( - nix_support.failed, - nix_support.hydra_release_name, - nix_support.products, - nix_support.metrics, - ) - } else { - (false, None, vec![], vec![]) - }; Ok(Self { - failed, + failed: merged.failed, timings: BuildTimings::new(v.import_time_ms, v.build_time_ms, v.upload_time_ms), - release_name, + release_name: merged.hydra_release_name, closure_size, size: nar_size, - products: products - .into_iter() - .map(|p| BuildProduct::from_grpc(store_dir, p)) - .collect::>>()?, + products: merged.products, outputs, - metrics: metrics - .into_iter() - .map(|v| BuildMetric { - name: v.name, - unit: v.unit, - value: v.value, - }) - .collect(), + metrics: merged.metrics, }) } } impl BuildOutput { - #[tracing::instrument(skip(store, outputs), err)] + #[tracing::instrument(skip(store, real_store_dir, outputs), err)] pub async fn new( - store: &nix_utils::LocalStore, - outputs: BTreeMap>, - ) -> anyhow::Result { + store: &harmonia_store_remote::ConnectionPool, + real_store_dir: &std::path::Path, + outputs: BTreeMap>, + ) -> Result { let resolved: BTreeMap<_, _> = outputs .iter() .filter_map(|(name, path)| Some((name.clone(), path.as_ref()?.clone()))) .collect(); - let pathinfos = store - .query_path_infos(&resolved.values().collect::>()) - .await; - let nix_support = - Box::pin(shared::parse_nix_support_from_outputs(store, &resolved)).await?; + let mut pathinfos = BTreeMap::new(); + for path in resolved.values() { + if let Some(info) = daemon_client_utils::query_path_info(store, path).await? { + pathinfos.insert(path.clone(), info); + } + } + let fs = nix_support::FilesystemOperations { + real_store_dir: real_store_dir.to_owned(), + }; + let per_output = Box::pin(nix_support::parse_nix_support_from_outputs( + store.store_dir(), + real_store_dir, + &fs, + &resolved, + )) + .await?; + + let mut merged = nix_support::NixSupport::default(); + for ns in per_output.into_values() { + merged.combine(ns); + } let mut outputs_map = BTreeMap::new(); let mut closure_size = 0; @@ -602,39 +485,26 @@ impl BuildOutput { for (name, path) in resolved { if let Some(info) = pathinfos.get(&path) { - closure_size += store.compute_closure_size(&path).await; + closure_size += daemon_client_utils::compute_closure_size(store, &path).await; nar_size += info.nar_size; outputs_map.insert(name, path); } } Ok(Self { - failed: nix_support.failed, + failed: merged.failed, timings: BuildTimings::default(), - release_name: nix_support.hydra_release_name, + release_name: merged.hydra_release_name, closure_size, size: nar_size, - products: nix_support - .products - .into_iter() - .map(|p| BuildProduct::from_shared(store.store_dir(), p)) - .collect::>>()?, + products: merged.products, outputs: outputs_map, - metrics: nix_support - .metrics - .into_iter() - .map(|v| BuildMetric { - name: v.name, - unit: v.unit, - value: v.value, - }) - .collect(), + metrics: merged.metrics, }) } } pub(super) fn get_mark_build_sccuess_data<'a>( - store: &nix_utils::LocalStore, b: &'a Arc, res: &'a BuildOutput, ) -> db::models::MarkBuildSuccessData<'a> { @@ -654,28 +524,8 @@ pub(super) fn get_mark_build_sccuess_data<'a>( .iter() .map(|(name, path)| (name.clone(), path.clone())) .collect(), - products: res - .products - .iter() - .map(|v| db::models::BuildProduct { - r#type: &v.r#type, - subtype: &v.subtype, - filesize: v.file_size.and_then(|v| i64::try_from(v).ok()), - sha256hash: v.sha256hash.as_deref(), - path: v.path.as_ref().map(|p| p.print(store.store_dir())), - name: &v.name, - defaultpath: v.default_path.as_deref(), - }) - .collect(), - metrics: res - .metrics - .iter() - .map(|m| db::models::BuildMetric { - name: &m.name, - unit: m.unit.as_deref(), - value: m.value, - }) - .collect(), + products: res.products.clone(), + metrics: res.metrics.clone(), } } diff --git a/subprojects/hydra-queue-runner/src/state/drv.rs b/subprojects/hydra-queue-runner/src/state/drv.rs index 681b683b2..d36527e11 100644 --- a/subprojects/hydra-queue-runner/src/state/drv.rs +++ b/subprojects/hydra-queue-runner/src/state/drv.rs @@ -1,6 +1,8 @@ use std::collections::BTreeSet; -use nix_utils::SingleDerivedPath; +use harmonia_store_derivation::derivation::Derivation; +use harmonia_store_derivation::derived_path::{OutputName, SingleDerivedPath}; +use harmonia_store_path::StorePath; /// Output names of intermediate derivations for a dynamic derivation /// dependency, stored in reverse order so that the next level to resolve @@ -12,10 +14,10 @@ use nix_utils::SingleDerivedPath; /// /// In the common case of depending on a static derivation, this is empty. #[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord)] -pub struct OutputNameChain(pub Vec); +pub struct OutputNameChain(pub Vec); impl OutputNameChain { - pub fn pop(&mut self) -> Option { + pub fn pop(&mut self) -> Option { self.0.pop() } } @@ -25,7 +27,8 @@ impl OutputNameChain { /// The output chain is in stack order (outermost first) matching /// [`OutputNameChain`]'s convention. For `Built { Opaque(A), "foo" }`, /// returns `(A, ["foo"])`. For `Opaque(A)`, returns `(A, [])`. -pub fn flatten_path(sdp: &SingleDerivedPath) -> (nix_utils::StorePath, OutputNameChain) { +#[must_use] +pub fn flatten_path(sdp: &SingleDerivedPath) -> (StorePath, OutputNameChain) { match sdp { SingleDerivedPath::Opaque(p) => (p.clone(), OutputNameChain::default()), SingleDerivedPath::Built { drv_path, output } => flatten_chain(drv_path, output), @@ -36,10 +39,11 @@ pub fn flatten_path(sdp: &SingleDerivedPath) -> (nix_utils::StorePath, OutputNam /// /// For `Built { Opaque(A), "foo" }` with output `"bar"`, /// returns `(A, ["bar", "foo"])`. +#[must_use] pub fn flatten_chain( drv_path: &SingleDerivedPath, - output_name: &nix_utils::OutputName, -) -> (nix_utils::StorePath, OutputNameChain) { + output_name: &OutputName, +) -> (StorePath, OutputNameChain) { let (root, mut chain) = flatten_path(drv_path); chain.0.push(output_name.clone()); (root, chain) @@ -51,9 +55,7 @@ pub fn flatten_chain( /// skipped — only derivation build dependencies are returned. For each /// `Built` input, the outermost output name (what we consume) is discarded; /// intermediate output names form the [`OutputNameChain`]. -pub fn input_drvs( - drv: &nix_utils::Derivation, -) -> BTreeSet<(nix_utils::StorePath, OutputNameChain)> { +pub fn input_drvs(drv: &Derivation) -> BTreeSet<(StorePath, OutputNameChain)> { drv.inputs .iter() .filter_map(|sdp| match sdp { diff --git a/subprojects/hydra-queue-runner/src/state/fod_checker.rs b/subprojects/hydra-queue-runner/src/state/fod_checker.rs index 2c0e50553..ec99646c1 100644 --- a/subprojects/hydra-queue-runner/src/state/fod_checker.rs +++ b/subprojects/hydra-queue-runner/src/state/fod_checker.rs @@ -1,10 +1,11 @@ use std::sync::Arc; +use harmonia_store_derivation::derivation::{Derivation, DerivationOutput}; +use harmonia_store_path::StorePath; use hashbrown::{HashMap, HashSet}; -use nix_utils::{Derivation, LocalStore, StorePath}; -#[derive(Debug)] pub struct FodChecker { + pool: harmonia_store_remote::ConnectionPool, ca_derivations: parking_lot::RwLock>, to_traverse: parking_lot::RwLock>, @@ -12,8 +13,23 @@ pub struct FodChecker { traverse_done_notifier: Option>, } +impl std::fmt::Debug for FodChecker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FodChecker") + .field( + "pool", + &format_args!("", self.pool.store_dir()), + ) + .field("ca_derivations", &self.ca_derivations) + .field("to_traverse", &self.to_traverse) + .field("notify_traverse", &self.notify_traverse) + .field("traverse_done_notifier", &self.traverse_done_notifier) + .finish() + } +} + async fn collect_ca_derivations( - store: &LocalStore, + store: &harmonia_store_remote::ConnectionPool, drv: &StorePath, processed: Arc>>, ) -> HashMap { @@ -30,16 +46,25 @@ async fn collect_ca_derivations( p.insert(drv.clone()); } - let Some(parsed) = nix_utils::query_drv(store, drv).await.ok().flatten() else { + let parsed = async { + let drv_path_str = store.store_dir().display(drv).to_string(); + let content = fs_err::tokio::read_to_string(&drv_path_str).await.ok()?; + let drv_name_str = drv.name().to_string(); + let name = drv_name_str.strip_suffix(".drv")?.parse().ok()?; + harmonia_store_aterm::parse_derivation_aterm(store.store_dir(), content.as_bytes(), name) + .ok() + } + .await; + let Some(parsed) = parsed as Option else { return HashMap::new(); }; let ca_fixed_hash = parsed.outputs.values().find_map(|o| match o { - nix_utils::DerivationOutput::CAFixed(ca) => Some(ca.hash()), + DerivationOutput::CAFixed(ca) => Some(ca.hash()), _ => None, }); let input_drvs: Vec = - harmonia_store_core::derivation::DerivationInputs::from(&parsed.inputs) + harmonia_store_derivation::derivation::DerivationInputs::from(&parsed.inputs) .drvs .into_keys() .collect(); @@ -64,8 +89,12 @@ async fn collect_ca_derivations( impl FodChecker { #[must_use] - pub fn new(traverse_done_notifier: Option>) -> Self { + pub fn new( + pool: harmonia_store_remote::ConnectionPool, + traverse_done_notifier: Option>, + ) -> Self { Self { + pool, ca_derivations: parking_lot::RwLock::new(HashMap::with_capacity(1000)), to_traverse: parking_lot::RwLock::new(HashSet::new()), @@ -76,7 +105,7 @@ impl FodChecker { pub(super) fn add_ca_drv_parsed(&self, drv: &StorePath, parsed: &Derivation) { let ca_fixed_hash = parsed.outputs.values().find_map(|o| match o { - nix_utils::DerivationOutput::CAFixed(ca) => Some(ca.hash()), + DerivationOutput::CAFixed(ca) => Some(ca.hash()), _ => None, }); if ca_fixed_hash.is_some() { @@ -90,7 +119,7 @@ impl FodChecker { tt.insert(drv.clone()); } - async fn traverse(&self, store: &LocalStore) { + async fn traverse(&self, store: &harmonia_store_remote::ConnectionPool) { use futures::StreamExt as _; let drvs = { @@ -127,10 +156,9 @@ impl FodChecker { loop { tokio::select! { () = self.notify_traverse.notified() => {}, - () = tokio::time::sleep(tokio::time::Duration::from_secs(60)) => {}, + () = tokio::time::sleep(tokio::time::Duration::from_mins(1)) => {}, }; - let store = LocalStore::init(); - self.traverse(&store).await; + self.traverse(&self.pool).await; if let Some(tx) = &self.traverse_done_notifier { let _ = tx.send(()).await; } @@ -170,19 +198,25 @@ mod tests { #![allow(clippy::unwrap_used)] use crate::state::fod_checker::FodChecker; - use nix_utils::BaseStore as _; + use harmonia_store_path::StorePath; #[ignore = "Requires a valid drv in the nix-store"] #[tokio::test] async fn test_traverse() { - let store = nix_utils::LocalStore::init(); + let nix_config = daemon_client_utils::parse_nix_remote().unwrap(); + let store = harmonia_store_remote::ConnectionPool::new( + &nix_config.socket, + harmonia_store_remote::PoolConfig::default(), + ); let hello_drv = - nix_utils::parse_store_path("rl5m4zxd24mkysmpbp4j9ak6q7ia6vj8-hello-2.12.2.drv"); - store.ensure_path(&hello_drv).await.unwrap(); + StorePath::from_base_path("rl5m4zxd24mkysmpbp4j9ak6q7ia6vj8-hello-2.12.2.drv").unwrap(); + daemon_client_utils::ensure_path(&store, &hello_drv) + .await + .unwrap(); - let fod = FodChecker::new(None); + let fod = FodChecker::new(store.clone(), None); fod.to_traverse(&hello_drv); - fod.traverse(&store).await; + fod.traverse(&fod.pool).await; assert_eq!(fod.ca_derivations.read().len(), 59); } } diff --git a/subprojects/hydra-queue-runner/src/state/jobset.rs b/subprojects/hydra-queue-runner/src/state/jobset.rs index 7d8a7d387..2cc7edcfe 100644 --- a/subprojects/hydra-queue-runner/src/state/jobset.rs +++ b/subprojects/hydra-queue-runner/src/state/jobset.rs @@ -5,6 +5,22 @@ use std::sync::atomic::{AtomicI64, AtomicU32, Ordering}; use hashbrown::HashMap; +#[derive(Debug, thiserror::Error)] +pub enum JobsetError { + #[error("database error")] + Db(#[from] db::Error), + + #[error("scheduling shares not found for jobset {jobset_id}")] + MissingShares { jobset_id: i32 }, + + #[error("scheduling shares out of range: {value}")] + SharesOutOfRange { + value: i32, + #[source] + source: std::num::TryFromIntError, + }, +} + pub type JobsetID = i32; pub(super) const SCHEDULING_WINDOW: i64 = 24 * 60 * 60; @@ -61,10 +77,8 @@ impl Jobset { ((seconds as f64) / f64::from(shares)) } - pub fn set_shares(&self, shares: i32) -> anyhow::Result<()> { - debug_assert!(shares > 0); - self.shares.store(shares.try_into()?, Ordering::Relaxed); - Ok(()) + pub fn set_shares(&self, shares: u32) { + self.shares.store(shares, Ordering::Relaxed); } pub fn get_shares(&self) -> u32 { @@ -84,10 +98,7 @@ impl Jobset { let now = jiff::Timestamp::now().as_second(); let mut steps = self.steps.write(); - loop { - let Some(first) = steps.first_entry() else { - break; - }; + while let Some(first) = steps.first_entry() { let start_time = *first.key(); if start_time > now - SCHEDULING_WINDOW { @@ -162,7 +173,7 @@ impl Jobsets { jobset_id: i32, project_name: &str, jobset_name: &str, - ) -> anyhow::Result> { + ) -> Result, JobsetError> { let key = (project_name.to_owned(), jobset_name.to_owned()); { let jobsets = self.inner.read(); @@ -174,9 +185,9 @@ impl Jobsets { let shares = conn .get_jobset_scheduling_shares(jobset_id) .await? - .ok_or_else(|| anyhow::anyhow!("Scheduling Shares not found for jobset not found."))?; + .ok_or(JobsetError::MissingShares { jobset_id })?; let jobset = Jobset::new(jobset_id, project_name, jobset_name); - jobset.set_shares(shares)?; + jobset.set_shares(shares); for step in conn .get_jobset_build_steps(jobset_id, SCHEDULING_WINDOW) @@ -201,20 +212,19 @@ impl Jobsets { } #[tracing::instrument(skip(self, conn), err)] - pub async fn handle_change(&self, conn: &mut db::Connection) -> anyhow::Result<()> { + pub async fn handle_change(&self, conn: &mut db::Connection) -> Result<(), JobsetError> { let curr_jobsets_in_db = conn.get_jobsets().await?; let jobsets = self.inner.read(); for row in curr_jobsets_in_db { - if let Some(i) = jobsets.get(&(row.project.clone(), row.name.clone())) - && let Err(e) = i.set_shares(row.schedulingshares) - { - tracing::error!( - "Failed to update jobset scheduling shares. project_name={} jobset_name={} e={}", - row.project, - row.name, - e, - ); + if let Some(i) = jobsets.get(&(row.project.clone(), row.name.clone())) { + let shares = u32::try_from(row.schedulingshares).map_err(|source| { + JobsetError::SharesOutOfRange { + value: row.schedulingshares, + source, + } + })?; + i.set_shares(shares); } } Ok(()) diff --git a/subprojects/hydra-queue-runner/src/state/machine.rs b/subprojects/hydra-queue-runner/src/state/machine.rs index 1851902fd..ffd3943b7 100644 --- a/subprojects/hydra-queue-runner/src/state/machine.rs +++ b/subprojects/hydra-queue-runner/src/state/machine.rs @@ -1,3 +1,4 @@ +use harmonia_store_path::StorePath; use std::sync::{Arc, atomic::Ordering}; use hashbrown::{HashMap, HashSet}; @@ -8,37 +9,23 @@ use db::models::BuildID; use super::{RemoteBuild, System}; use crate::config::{MachineFreeFn, MachineSortFn}; -use crate::server::grpc::runner_v1::{AbortMessage, BuildMessage, JoinMessage, runner_request}; - -#[derive(Debug, Clone, Copy)] -pub struct Pressure { - pub avg10: f32, - pub avg60: f32, - pub avg300: f32, - pub total: u64, -} +use hydra_proto::{AbortMessage, BuildMessage, JoinMessage, PresignedUploadOpts, runner_request}; -impl From for Pressure { - fn from(v: crate::server::grpc::runner_v1::Pressure) -> Self { - Self { - avg10: v.avg10, - avg60: v.avg60, - avg300: v.avg300, - total: v.total, - } - } -} +#[derive(Debug, thiserror::Error)] +pub enum MachineError { + #[error("{0}")] + ConfigIncompat(String), + + #[error("failed to send message to machine: channel closed")] + Channel(#[from] mpsc::error::SendError), -#[derive(Debug, Clone, Copy)] -pub struct PressureState { - pub cpu_some: Option, - pub mem_some: Option, - pub mem_full: Option, - pub io_some: Option, - pub io_full: Option, - pub irq_full: Option, + #[error("parsing machine UUID")] + Uuid(#[from] uuid::Error), } +pub use hydra_proto::Pressure; +pub(crate) use hydra_proto::PressureState; + #[derive(Debug)] pub struct Stats { current_jobs: std::sync::atomic::AtomicU64, @@ -222,7 +209,7 @@ impl Stats { self.last_ping.load(Ordering::Relaxed) } - pub fn store_ping(&self, msg: &crate::server::grpc::runner_v1::PingMessage) { + pub fn store_ping(&self, msg: &hydra_proto::PingMessage) { self.last_ping .store(jiff::Timestamp::now().as_second(), Ordering::Relaxed); @@ -232,14 +219,7 @@ impl Stats { self.mem_usage.store(msg.mem_usage, Ordering::Relaxed); if let Some(p) = msg.pressure { - self.pressure.store(Some(Arc::new(PressureState { - cpu_some: p.cpu_some.map(Into::into), - mem_some: p.mem_some.map(Into::into), - mem_full: p.mem_full.map(Into::into), - io_some: p.io_some.map(Into::into), - io_full: p.io_full.map(Into::into), - irq_full: p.irq_full.map(Into::into), - }))); + self.pressure.store(Some(Arc::new(p))); } self.build_dir_free_percent @@ -490,14 +470,14 @@ impl Machines { #[derive(Debug, Clone)] pub struct Job { pub internal_build_id: uuid::Uuid, - pub path: nix_utils::StorePath, + pub path: StorePath, pub build_id: BuildID, pub step_nr: i32, pub result: RemoteBuild, } impl Job { - pub fn new(build_id: BuildID, path: nix_utils::StorePath) -> Self { + pub fn new(build_id: BuildID, path: StorePath) -> Self { Self { internal_build_id: uuid::Uuid::new_v4(), path, @@ -508,34 +488,19 @@ impl Job { } } -#[derive(Debug, Clone, Copy)] -pub struct PresignedUrlOpts { - pub upload_debug_info: bool, -} - -impl From for crate::server::grpc::runner_v1::PresignedUploadOpts { - fn from(value: PresignedUrlOpts) -> Self { - Self { - upload_debug_info: value.upload_debug_info, - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct ConfigUpdate { - pub max_concurrent_downloads: u32, -} +pub(crate) use hydra_proto::ConfigUpdate; #[derive(Debug)] pub enum Message { ConfigUpdate(ConfigUpdate), BuildMessage { build_id: uuid::Uuid, - drv: nix_utils::StorePath, + drv: StorePath, max_log_size: u64, max_silent_time: i32, build_timeout: i32, - presigned_url_opts: Option, + presigned_url_opts: Option, + resolved_drv: Box, }, AbortMessage { build_id: uuid::Uuid, @@ -543,13 +508,10 @@ pub enum Message { } impl Message { - pub fn into_request(self) -> crate::server::grpc::runner_v1::RunnerRequest { + #[must_use] + pub fn into_request(self) -> hydra_proto::RunnerRequest { let msg = match self { - Self::ConfigUpdate(m) => runner_request::Message::ConfigUpdate( - crate::server::grpc::runner_v1::ConfigUpdate { - max_concurrent_downloads: m.max_concurrent_downloads, - }, - ), + Self::ConfigUpdate(m) => runner_request::Message::ConfigUpdate(m), Self::BuildMessage { build_id, drv, @@ -557,20 +519,22 @@ impl Message { max_silent_time, build_timeout, presigned_url_opts, + resolved_drv, } => runner_request::Message::Build(BuildMessage { build_id: build_id.to_string(), - drv: Some(shared::proto::ProtoStorePath::from(drv)), + drv: Some(hydra_proto::ProtoStorePath::from(drv)), max_log_size, max_silent_time, build_timeout, - presigned_url_opts: presigned_url_opts.map(Into::into), + presigned_url_opts, + resolved_drv: Some(*resolved_drv), }), Self::AbortMessage { build_id } => runner_request::Message::Abort(AbortMessage { build_id: build_id.to_string(), }), }; - crate::server::grpc::runner_v1::RunnerRequest { message: Some(msg) } + hydra_proto::RunnerRequest { message: Some(msg) } } } @@ -631,21 +595,20 @@ impl Machine { tx: mpsc::Sender, use_presigned_uploads: bool, forced_substituters: &[String], - ) -> anyhow::Result { + ) -> Result { if use_presigned_uploads && !forced_substituters.is_empty() { if !msg.use_substitutes { - return Err(anyhow::anyhow!( - "Forced_substituters is configured but builder doesnt use substituters. This is an issue because presigned uploads are enabled", + return Err(MachineError::ConfigIncompat( + "Forced_substituters is configured but builder doesnt use substituters. This is an issue because presigned uploads are enabled".into(), )); } for forced_sub in forced_substituters { if !msg.substituters.contains(forced_sub) { - return Err(anyhow::anyhow!( + return Err(MachineError::ConfigIncompat(format!( "Builder missing required substituter '{}'. Available: {:?}", - forced_sub, - msg.substituters - )); + forced_sub, msg.substituters + ))); } } } @@ -680,26 +643,34 @@ impl Machine { } #[tracing::instrument( - skip(self, job, opts, presigned_url_opts), + skip(self, job, presigned_url_opts), fields(build_id=job.build_id, step_nr=job.step_nr), err, )] + #[expect( + clippy::too_many_arguments, + reason = "params mirror the fields dispatched in `Message::BuildMessage`" + )] pub async fn build_drv( &self, job: Job, - effective_drv: nix_utils::StorePath, - opts: &nix_utils::BuildOptions, - presigned_url_opts: Option, - ) -> anyhow::Result<()> { + effective_drv: StorePath, + max_log_size: u64, + max_silent_time: i32, + build_timeout: i32, + presigned_url_opts: Option, + resolved_drv: hydra_proto::nix::store::derivation::v1::Basic, + ) -> Result<(), MachineError> { let drv = effective_drv; self.msg_queue .send(Message::BuildMessage { build_id: job.internal_build_id, drv, - max_log_size: opts.get_max_log_size(), - max_silent_time: opts.get_max_silent_time(), - build_timeout: opts.get_build_timeout(), + max_log_size, + max_silent_time, + build_timeout, presigned_url_opts, + resolved_drv: Box::new(resolved_drv), }) .await?; @@ -718,7 +689,7 @@ impl Machine { } #[tracing::instrument(skip(self), fields(build_id=%build_id), err)] - pub async fn abort_build(&self, build_id: uuid::Uuid) -> anyhow::Result<()> { + pub async fn abort_build(&self, build_id: uuid::Uuid) -> Result<(), MachineError> { self.msg_queue .send(Message::AbortMessage { build_id }) .await?; @@ -728,7 +699,7 @@ impl Machine { } #[tracing::instrument(skip(self), err)] - pub async fn publish_config_update(&self, change: ConfigUpdate) -> anyhow::Result<()> { + pub async fn publish_config_update(&self, change: ConfigUpdate) -> Result<(), MachineError> { self.msg_queue.send(Message::ConfigUpdate(change)).await?; Ok(()) } @@ -832,7 +803,7 @@ impl Machine { } #[tracing::instrument(skip(self), fields(%drv))] - pub fn get_build_id_and_step_nr(&self, drv: &nix_utils::StorePath) -> Option<(i32, i32)> { + pub fn get_build_id_and_step_nr(&self, drv: &StorePath) -> Option<(i32, i32)> { let jobs = self.jobs.read(); jobs.iter() .find(|j| &j.path == drv) @@ -848,7 +819,7 @@ impl Machine { } #[tracing::instrument(skip(self), fields(%build_id))] - pub fn get_job_drv_for_build_id(&self, build_id: uuid::Uuid) -> Option { + pub fn get_job_drv_for_build_id(&self, build_id: uuid::Uuid) -> Option { let jobs = self.jobs.read(); jobs.iter() .find(|j| j.internal_build_id == build_id) @@ -856,7 +827,7 @@ impl Machine { } #[tracing::instrument(skip(self), fields(%drv))] - pub fn get_internal_build_id_for_drv(&self, drv: &nix_utils::StorePath) -> Option { + pub fn get_internal_build_id_for_drv(&self, drv: &StorePath) -> Option { let jobs = self.jobs.read(); jobs.iter() .find(|j| &j.path == drv) @@ -871,7 +842,7 @@ impl Machine { } #[tracing::instrument(skip(self), fields(%drv))] - pub fn remove_job(&self, drv: &nix_utils::StorePath) -> Option { + pub fn remove_job(&self, drv: &StorePath) -> Option { let job = { let mut jobs = self.jobs.write(); let job = jobs.iter().find(|j| &j.path == drv).cloned(); diff --git a/subprojects/hydra-queue-runner/src/state/metrics.rs b/subprojects/hydra-queue-runner/src/state/metrics.rs index 6ab4b63cc..b593356e6 100644 --- a/subprojects/hydra-queue-runner/src/state/metrics.rs +++ b/subprojects/hydra-queue-runner/src/state/metrics.rs @@ -2,8 +2,6 @@ use std::sync::Arc; use prometheus::Encoder as _; -use nix_utils::BaseStore as _; - #[derive(Debug)] pub struct PromMetrics { pub queue_runner_current_time_seconds: prometheus::IntGauge, // hydraqueuerunner_current_time_seconds @@ -122,7 +120,7 @@ pub struct PromMetrics { impl PromMetrics { #[allow(clippy::too_many_lines)] #[tracing::instrument(err)] - pub fn new() -> anyhow::Result { + pub fn new() -> Result { let queue_checks_started = prometheus::IntCounter::with_opts(prometheus::Opts::new( "hydraqueuerunner_queue_checks_started_total", "Number of times State::get_queued_builds() was started", @@ -852,12 +850,23 @@ impl PromMetrics { pub async fn refresh_dynamic_metrics(&self, state: &Arc) { let nr_steps_done = self.nr_steps_done.get(); - if nr_steps_done > 0 { - let avg_time = self.total_step_time_ms.get() / nr_steps_done; - let avg_import_time = self.total_step_import_time_ms.get() / nr_steps_done; - let avg_build_time = self.total_step_build_time_ms.get() / nr_steps_done; - let avg_upload_time = self.total_step_upload_time_ms.get() / nr_steps_done; - + if let ( + Some(avg_time), + Some(avg_import_time), + Some(avg_build_time), + Some(avg_upload_time), + ) = ( + self.total_step_time_ms.get().checked_div(nr_steps_done), + self.total_step_import_time_ms + .get() + .checked_div(nr_steps_done), + self.total_step_build_time_ms + .get() + .checked_div(nr_steps_done), + self.total_step_upload_time_ms + .get() + .checked_div(nr_steps_done), + ) { if let Ok(v) = i64::try_from(avg_time) { self.avg_step_time_ms.set(v); } @@ -890,7 +899,6 @@ impl PromMetrics { self.refresh_per_machine_type_metrics(state).await; self.refresh_per_machine_metrics(state); - self.refresh_store_metrics(state); self.refresh_s3_metrics(state); self.refresh_transfer_metrics(state); self.refresh_jobset_metrics(state); @@ -996,54 +1004,6 @@ impl PromMetrics { } } - fn refresh_store_metrics(&self, state: &Arc) { - if let Ok(store_stats) = state.store.get_store_stats() { - if let Ok(v) = i64::try_from(store_stats.nar_info_read) { - self.store_nar_info_read.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_info_read_averted) { - self.store_nar_info_read_averted.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_info_missing) { - self.store_nar_info_missing.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_info_write) { - self.store_nar_info_write.set(v); - } - if let Ok(v) = i64::try_from(store_stats.path_info_cache_size) { - self.store_path_info_cache_size.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_read) { - self.store_nar_read.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_read_bytes) { - self.store_nar_read_bytes.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_read_compressed_bytes) { - self.store_nar_read_compressed_bytes.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_write) { - self.store_nar_write.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_write_averted) { - self.store_nar_write_averted.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_write_bytes) { - self.store_nar_write_bytes.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_write_compressed_bytes) { - self.store_nar_write_compressed_bytes.set(v); - } - if let Ok(v) = i64::try_from(store_stats.nar_write_compression_time_ms) { - self.store_nar_write_compression_time_ms.set(v); - } - self.store_nar_compression_savings - .set(store_stats.nar_compression_savings()); - self.store_nar_compression_speed - .set(store_stats.nar_compression_speed()); - } - } - fn refresh_s3_metrics(&self, state: &Arc) { self.s3_put.reset(); self.s3_put_bytes.reset(); @@ -1063,7 +1023,7 @@ impl PromMetrics { let backends = state.remote_stores.read(); for remote_store in backends.iter().filter_map(|s| match s { super::RemoteStoreBackend::S3(s) => Some(s), - _ => None, + super::RemoteStoreBackend::NixCopy(_) => None, }) { let backend_name = &remote_store.cfg.client_config.bucket; let s3_stats = remote_store.s3_stats(); @@ -1149,7 +1109,10 @@ impl PromMetrics { } #[tracing::instrument(skip(self, state), err)] - pub async fn gather_metrics(&self, state: &Arc) -> anyhow::Result> { + pub async fn gather_metrics( + &self, + state: &Arc, + ) -> Result, prometheus::Error> { self.refresh_dynamic_metrics(state).await; let mut buffer = Vec::new(); diff --git a/subprojects/hydra-queue-runner/src/state/mod.rs b/subprojects/hydra-queue-runner/src/state/mod.rs index c2e3b5f78..ad8819725 100644 --- a/subprojects/hydra-queue-runner/src/state/mod.rs +++ b/subprojects/hydra-queue-runner/src/state/mod.rs @@ -11,16 +11,106 @@ mod step; mod step_info; mod uploader; -use anyhow::Context as _; pub use atomic::AtomicDateTime; + +/// Errors from external subsystems, plus the combined state-logic errors +/// under [`Logic`](`Self::Logic`). +#[derive(Debug, thiserror::Error)] +pub enum StateError { + #[error("database error")] + Db(#[from] db::Error), + + #[error("I/O error")] + Io(#[from] std::io::Error), + + #[error("nix daemon error")] + Daemon(#[from] harmonia_store_remote::DaemonError), + + #[error("jobset error")] + Jobset(#[from] jobset::JobsetError), + + #[error("build output error")] + BuildOutput(#[from] build::BuildOutputError), + + #[error("machine error")] + Machine(#[from] machine::MachineError), + + #[error("metrics error")] + Metrics(#[from] prometheus::Error), + + #[error("configuration error")] + Config(#[from] crate::config::ConfigError), + + #[error("binary cache error")] + Cache(#[from] binary_cache::CacheError), + + #[error("integer conversion error")] + IntConversion(#[from] std::num::TryFromIntError), + + #[error("time computation error")] + Jiff(#[from] jiff::Error), + + #[error("invalid platform UTF-8: {0}")] + InvalidPlatformUtf8(std::str::Utf8Error), + + #[error("failed to construct log path string")] + LogPathNotUtf8, + + #[error("reading derivation `{drv}`: {reason}")] + ReadDerivation { + drv: StorePath, + reason: &'static str, + }, + + #[error("state logic error")] + Logic(#[from] StateLogicError), +} + +/// All state logic errors combined. Sub-enums are defined alongside +/// the `impl State` blocks that use them. +#[derive(Debug, thiserror::Error)] +pub enum StateLogicError { + #[error(transparent)] + Resolution(#[from] ResolutionError), + #[error(transparent)] + StepLookup(#[from] StepLookupError), + #[error(transparent)] + MachineLookup(#[from] MachineLookupError), + #[error(transparent)] + DrvLookup(#[from] DrvLookupError), +} + +impl From for StateError { + fn from(e: ResolutionError) -> Self { + Self::Logic(StateLogicError::Resolution(e)) + } +} +impl From for StateError { + fn from(e: StepLookupError) -> Self { + Self::Logic(StateLogicError::StepLookup(e)) + } +} +impl From for StateError { + fn from(e: MachineLookupError) -> Self { + Self::Logic(StateLogicError::MachineLookup(e)) + } +} +impl From for StateError { + fn from(e: DrvLookupError) -> Self { + Self::Logic(StateLogicError::DrvLookup(e)) + } +} + pub use build::{Build, BuildOutput, BuildResultState, BuildTimings, Builds, RemoteBuild}; +use harmonia_store_derivation::derivation::Derivation; +use harmonia_store_path::StorePath; pub use jobset::{Jobset, JobsetID, Jobsets}; pub use machine::{Machine, Message as MachineMessage, Pressure, Stats as MachineStats}; pub use queue::{BuildQueueStats, Queues}; pub use step::{Step, Steps}; pub use step_info::StepInfo; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use std::sync::atomic::{AtomicI64, Ordering}; use std::time::Instant; @@ -30,8 +120,10 @@ use hashbrown::{HashMap, HashSet}; use secrecy::ExposeSecret as _; use db::models::{BuildID, BuildStatus}; +use harmonia_store_derivation::derivation::DerivationOutput; +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_derivation::realisation::{DrvOutput, Realisation, UnkeyedRealisation}; use inspectable_channel::InspectableChannel; -use nix_utils::BaseStore as _; use crate::config::{App, Cli}; use crate::state::build::get_mark_build_sccuess_data; @@ -40,6 +132,7 @@ use crate::state::machine::Machines; use crate::utils::finish_build_step; pub type System = String; +const MAX_CONCURRENT_BUILD_INJECTION: usize = 10; enum CreateStepResult { None, @@ -56,15 +149,23 @@ enum RealiseStepResult { CachedFailure, } +struct ProcessedBuild { + _id: BuildID, + nr_added: Arc, + new_runnable: Arc>>>, + elapsed: u64, +} + #[allow(missing_debug_implementations)] pub enum RemoteStoreBackend { S3(binary_cache::S3BinaryCacheClient), - Nix(nix_utils::RemoteStore), + /// A nix store reachable via `nix copy --to `. + NixCopy(String), } #[allow(missing_debug_implementations)] pub struct State { - pub store: nix_utils::LocalStore, + pub pool: harmonia_store_remote::ConnectionPool, pub remote_stores: parking_lot::RwLock>, pub config: App, pub cli: Cli, @@ -86,7 +187,7 @@ pub struct State { /// /// FIXME: Replace this with proper persisted column, so we don't have to re-resolve on /// restart. - pub resolved_drv_map: parking_lot::RwLock>, + pub resolved_drv_map: parking_lot::RwLock>, pub fod_checker: Option>, @@ -95,14 +196,62 @@ pub struct State { pub metrics: metrics::PromMetrics, pub notify_dispatch: tokio::sync::Notify, pub uploader: uploader::Uploader, + + /// Parsed nix daemon store config (for reconstructing URIs etc). + pub nix_daemon_config: daemon_client_utils::NixDaemonStoreConfig, + /// Physical store directory on disk (for chroot stores). + /// Cached from `nix_daemon_config.real_store_dir()`. + /// `None` means the logical store dir is the filesystem path. + pub real_store_dir: Option, } impl State { + /// Resolve a store path to a filesystem path, accounting for chroot stores. + fn real_path(&self, path: &StorePath) -> std::path::PathBuf { + match &self.real_store_dir { + Some(real) => real.join(path.to_string()), + None => std::path::PathBuf::from(self.pool.store_dir().display(path).to_string()), + } + } + + /// Read and parse a `.drv` file from the store. + async fn read_derivation(&self, drv_path: &StorePath) -> Result { + let content = fs_err::tokio::read_to_string(self.real_path(drv_path)).await?; + let drv_name_str = drv_path.name().to_string(); + let name = drv_name_str + .strip_suffix(".drv") + .ok_or_else(|| StateError::ReadDerivation { + drv: drv_path.clone(), + reason: "name does not end in `.drv`", + })? + .parse() + .map_err(|_| StateError::ReadDerivation { + drv: drv_path.clone(), + reason: "parsing derivation name", + })?; + harmonia_store_aterm::parse_derivation_aterm( + self.pool.store_dir(), + content.as_bytes(), + name, + ) + .map_err(|_| StateError::ReadDerivation { + drv: drv_path.clone(), + reason: "parsing derivation ATerm", + }) + } + #[tracing::instrument(err)] - pub async fn new() -> anyhow::Result> { - let store = nix_utils::LocalStore::init(); - nix_utils::set_verbosity(1); - tracing::info!("LocalStore dir={}", nix_utils::get_store_dir()); + pub async fn new() -> Result, StateError> { + let nix_config = daemon_client_utils::parse_nix_remote() + .map_err(crate::config::ConfigError::ParseNixStore)?; + let store_dir = nix_config.store_dir.clone(); + let pool = harmonia_store_remote::ConnectionPool::with_store_dir( + &nix_config.socket, + store_dir.clone(), + harmonia_store_remote::PoolConfig::default(), + ); + + tracing::info!("LocalStore dir={store_dir}"); let cli = Cli::new(); @@ -126,13 +275,18 @@ impl State { binary_cache::S3BinaryCacheClient::new(cfg).await?, )); } else { - tracing::info!("Opening FFI store for: {uri}"); - remote_stores.push(RemoteStoreBackend::Nix(nix_utils::RemoteStore::init(&uri))); + remote_stores.push(RemoteStoreBackend::NixCopy(uri.clone())); } } + let fod_checker = if config.get_enable_fod_checker() { + Some(Arc::new(FodChecker::new(pool.clone(), None))) + } else { + None + }; + Ok(Arc::new(Self { - store, + pool, remote_stores: parking_lot::RwLock::new(remote_stores), cli, db, @@ -143,11 +297,7 @@ impl State { jobsets: Jobsets::new(), steps: Steps::new(), queues: Queues::new(), - fod_checker: if config.get_enable_fod_checker() { - Some(Arc::new(FodChecker::new(None))) - } else { - None - }, + fod_checker, started_at: jiff::Timestamp::now(), metrics: metrics::PromMetrics::new()?, notify_dispatch: tokio::sync::Notify::new(), @@ -155,6 +305,8 @@ impl State { config.get_hydra_data_dir().join("uploader_state.json"), ) .await, + real_store_dir: nix_config.real_store_dir(), + nix_daemon_config: nix_config, config, })) } @@ -163,7 +315,7 @@ impl State { pub async fn reload_config_callback( &self, new_config: &crate::config::PreparedApp, - ) -> anyhow::Result<()> { + ) -> Result<(), StateError> { // IF this gets more complex we need a way to trap the state and revert. // right now it doesnt matter because only reconfigure_pool can fail and this is the first // thing we do. @@ -181,9 +333,7 @@ impl State { binary_cache::S3BinaryCacheClient::new(cfg).await?, )); } else { - tracing::info!("Opening FFI store for: {uri}"); - new_remote_stores - .push(RemoteStoreBackend::Nix(nix_utils::RemoteStore::init(uri))); + new_remote_stores.push(RemoteStoreBackend::NixCopy(uri.clone())); } } } @@ -246,7 +396,9 @@ impl State { &job.path, // we fail this with preparing because we kinda want to restart all jobs if // a machine is removed - BuildResultState::PreparingFailure, + BuildResultState::Completed( + hydra_proto::BuildResultState::PreparingFailure, + ), BuildTimings::default(), ) .await @@ -267,25 +419,56 @@ impl State { } #[tracing::instrument(skip(self), err)] - pub async fn clear_busy(&self) -> anyhow::Result<()> { + pub async fn clear_busy(&self) -> Result<(), db::Error> { let mut db = self.db.get().await?; db.clear_busy(0).await?; Ok(()) } +} +/// Errors from derivation resolution (CA floating, dynamic deps). +#[derive(Debug, thiserror::Error)] +pub enum ResolutionError { + #[error("failed to resolve CAFloating derivation {0}")] + UnresolvedCAFloating(StorePath), + + #[error("failed to resolve derivation {0}")] + ResolveFailed(StorePath), + + #[error("failed to fill deferred outputs for {0}")] + FillDeferredOutputs(StorePath), + + #[error("could not create resolved build step")] + ResolvedStepCreationFailed, + + #[error("output path mismatch for output `{name}` of {drv}: expected {expected}, got {actual}")] + OutputPathMismatch { + name: String, + drv: StorePath, + expected: String, + actual: String, + }, + + #[error("dynamic rdep references output `{output}` not produced by {drv}")] + DynRdepOutputMissing { output: String, drv: StorePath }, +} + +impl State { #[tracing::instrument(skip(self, constraint), err)] #[allow(clippy::too_many_lines)] async fn realise_drv_on_valid_machine( self: Arc, constraint: queue::JobConstraint, - ) -> anyhow::Result { + ) -> Result { let free_fn = self.config.get_machine_free_fn(); let Some((machine, step_info)) = constraint.resolve(&self.machines, free_fn) else { return Ok(RealiseStepResult::None); }; let drv = step_info.step.get_drv_path(); - let mut build_options = nix_utils::BuildOptions::new(None); + let default_max_log_size: u64 = 64 << 20; // 64 MiB + let max_silent_time: i32; + let build_timeout: i32; let build = { let mut dependents = HashSet::new(); @@ -317,9 +500,8 @@ impl State { let biggest_max_silent_time = dependents.iter().map(|x| x.max_silent_time).max(); let biggest_build_timeout = dependents.iter().map(|x| x.timeout).max(); - build_options - .set_max_silent_time(biggest_max_silent_time.unwrap_or(build.max_silent_time)); - build_options.set_build_timeout(biggest_build_timeout.unwrap_or(build.timeout)); + max_silent_time = biggest_max_silent_time.unwrap_or(build.max_silent_time); + build_timeout = biggest_build_timeout.unwrap_or(build.timeout); build.clone() }; @@ -335,9 +517,7 @@ impl State { } self.construct_log_file_path(drv) - .await? - .to_str() - .ok_or_else(|| anyhow::anyhow!("failed to construct log path string."))? + .await .clone_into(&mut job.result.log_file); let mut db = self.db.get().await?; let step_nr = { @@ -345,7 +525,7 @@ impl State { let step_nr = tx .create_build_step( - self.store.store_dir(), + self.pool.store_dir(), Some(job.result.get_start_time_as_i32()?), build_id, step_info.step.get_drv_path(), @@ -356,7 +536,7 @@ impl State { None, step_info .step - .get_output_paths(self.store.store_dir()) + .get_output_paths(self.pool.store_dir()) .unwrap_or_default() .into_iter() .collect(), @@ -368,63 +548,82 @@ impl State { }; job.step_nr = step_nr; - // Try to resolve CA derivation inputs. If resolution yields a - // different drv, mark this step as Resolved in the DB and create a new - // step for the resolved drv that will be built at a later time. - { - let Some(guard) = step_info.step.get_drv() else { + // Resolve derivation inputs: replace `Built` input references with + // concrete output store paths using `try_resolve_force`. For + // input-addressed drvs this keeps the original output paths + // (the "force" part); for CA floating drvs it doesn't matter + // since there are no `InputAddressed` outputs. CA floating drvs + // that resolve to a different drv path still need the two-phase + // build dance. + let (basic_drv, was_deferred) = { + let guard = step_info.step.get_drv(); + let Some(drv_ref) = guard.as_ref() else { // Should never happen return Ok(RealiseStepResult::MaybeCancelled); }; - let drv_ref = guard.as_ref().unwrap(); - - // `Some(path)` if this CA derivation was resolved to a - // different drv; `None` if resolution is not needed or is - // a no-op (same drv path). - let resolved_path = async { - // Only CA floating derivations need resolution, and only - // when they have `Built` inputs (dynamic derivation deps). - // `Opaque`-only inputs are already resolved. - let is_ca_with_built_inputs = - drv_ref.outputs.iter().all(|output| { - matches!(output.1, nix_utils::DerivationOutput::CAFloating(_)) - }) && drv_ref - .inputs - .iter() - .any(|input| matches!(input, nix_utils::SingleDerivedPath::Built { .. })); - tracing::info!( - is_ca_with_built_inputs, - ca_floating = drv_ref - .outputs - .iter() - .all(|o| matches!(o.1, nix_utils::DerivationOutput::CAFloating(_))), - has_built_inputs = drv_ref - .inputs - .iter() - .any(|i| matches!(i, nix_utils::SingleDerivedPath::Built { .. })), - "resolution check for {drv}" - ); - if !is_ca_with_built_inputs { - return Ok::<_, anyhow::Error>(None); - } - // Resolve `Built` input placeholders to concrete store - // paths using outputs recorded in the DB. - let resolved_map = self.resolved_drv_map.read().clone(); - let basic_drv = - StepInfo::try_resolve(self.store.store_dir(), &self.db, drv_ref, &resolved_map) - .await - .ok_or_else(|| { - anyhow::anyhow!("Failed to resolve CAFloating derivation {drv}") - })?; - let resolved_path = self.store.write_derivation(&basic_drv).await?; - // If resolution changed the drv, we need a two-phase - // build; otherwise just build the original directly. - Ok((&resolved_path != drv).then_some(resolved_path)) + // Resolve `Built` input references to concrete store paths. + let resolved_map = self.resolved_drv_map.read().clone(); + let mut basic_drv = StepInfo::try_resolve_force( + self.pool.store_dir(), + &self.db, + drv_ref, + &resolved_map, + ) + .await + .ok_or_else(|| ResolutionError::ResolveFailed(drv.clone()))?; + + // Input-addressed outputs that transitively depend on a CA + // derivation come out of eval as `Deferred` because the IA + // hash can't be computed until the CA dep's path is known. + // Now that inputs are resolved, fill in the IA paths and + // matching `$out` env vars via `fill_outputs`. The resolved + // drv is then routed through the same two-phase dance as + // CAFloating below so we can detect if resolution changed + // the drv path. + let was_deferred = !basic_drv.outputs.is_empty() + && basic_drv + .outputs + .values() + .any(|o| matches!(o, DerivationOutput::Deferred)); + if was_deferred { + let unfilled = basic_drv + .clone() + .map_outputs(|_| harmonia_store_aterm::input_address::UnfilledOutput); + let filled = harmonia_store_aterm::input_address::fill_outputs( + self.pool.store_dir(), + unfilled, + ) + .map_err(|_| ResolutionError::FillDeferredOutputs(drv.clone()))?; + basic_drv = filled.map_outputs(DerivationOutput::InputAddressed); } - .await?; + (basic_drv, was_deferred) + }; - if let Some(resolved_path) = resolved_path { + // CA floating derivations and originally-Deferred derivations + // both need a two-phase build: write the resolved drv to the + // store via the daemon, get its assigned path, and (if it + // differs from the original) dispatch a new step for it. + if was_deferred + || basic_drv + .outputs + .iter() + .any(|o| matches!(o.1, DerivationOutput::CAFloating(_))) + { + // Write the resolved derivation to the store via daemon + // protocol so we can compare its path. + let resolved_path = { + let mut guard = self.pool.acquire().await?; + harmonia_protocol::daemon::write_derivation( + guard.client(), + self.pool.store_dir(), + &basic_drv, + false, + ) + .await? + .path + }; + if &resolved_path != drv { tracing::info!("resolved CA derivation {drv} -> {resolved_path}"); // Record the resolved drv path in memory so future @@ -442,7 +641,7 @@ impl State { resolved_result.log_file.clone_from(&job.result.log_file); finish_build_step( &self.db, - &self.store, + self.pool.store_dir(), build_id, step_nr, &resolved_result, @@ -451,6 +650,16 @@ impl State { ) .await?; + // Only mark the resolved step as a direct build if the + // unresolved step was the toplevel derivation of the build. + // Otherwise, `succeed_step` would prematurely mark the build as + // finished when an intermediate resolved step completes. + let is_toplevel = build.toplevel.load().as_deref() == Some(&*step_info.step); + let referring_build = if is_toplevel { + Some(build.clone()) + } else { + None + }; // Create a resolved in-memory step // We do not need the global state because they are only relevant when making // multiple steps. Our resolved step, by definition, has no dependencies, so @@ -463,16 +672,6 @@ impl State { // we can take the return of the function. // new_runnable: An output list of the runnable steps. Again, only one will be made, so we // can take the function return. - // Only mark the resolved step as a direct build if the - // unresolved step was the toplevel derivation of the build. - // Otherwise, `succeed_step` would prematurely mark the build as - // finished when an intermediate resolved step completes. - let is_toplevel = build.toplevel.load().as_deref() == Some(&*step_info.step); - let referring_build = if is_toplevel { - Some(build.clone()) - } else { - None - }; let resolved_step = match self .create_step( build.clone(), @@ -486,17 +685,14 @@ impl State { .await { CreateStepResult::None => { - return Err(anyhow::anyhow!("Could not create resolved build step")); + return Err(StateError::from( + ResolutionError::ResolvedStepCreationFailed, + )); } CreateStepResult::Valid(step) => step, CreateStepResult::PreviousFailure(step) => { self.handle_previous_failure(build.clone(), step.clone()) - .await - .with_context(|| { - format!( - "Failed to handle previous failure in resolved version of {drv}" - ) - })?; + .await?; return Ok(RealiseStepResult::CachedFailure); } }; @@ -542,6 +738,9 @@ impl State { } } + // Encode the force-resolved derivation as protobuf to send to the builder. + let resolved_drv = hydra_proto::nix::store::derivation::v1::Basic::from(&basic_drv); + { let mut tx = db.begin_transaction().await?; tx.notify_build_started(build_id).await?; @@ -566,19 +765,22 @@ impl State { .build_drv( job, drv.clone(), - &build_options, + default_max_log_size, + max_silent_time, + build_timeout, // TODO: cleanup if self.config.use_presigned_uploads() { let remote_stores = self.remote_stores.read(); remote_stores.iter().find_map(|s| match s { - RemoteStoreBackend::S3(s) => Some(machine::PresignedUrlOpts { + RemoteStoreBackend::S3(s) => Some(hydra_proto::PresignedUploadOpts { upload_debug_info: s.cfg.write_debug_info, }), - _ => None, + RemoteStoreBackend::NixCopy(_) => None, }) } else { None }, + resolved_drv, ) .await?; self.metrics.nr_steps_started.inc(); @@ -586,74 +788,118 @@ impl State { Ok(RealiseStepResult::Valid(machine)) } - #[tracing::instrument(skip(self), fields(%drv), err)] - async fn construct_log_file_path( - &self, - drv: &nix_utils::StorePath, - ) -> anyhow::Result { + #[tracing::instrument(skip(self), fields(%drv))] + async fn construct_log_file_path(&self, drv: &StorePath) -> std::path::PathBuf { let mut log_file = self.log_dir.clone(); let base = drv.to_string(); let (dir, file) = base.split_at(2); log_file.push(format!("{dir}/")); - let _ = fs_err::tokio::create_dir_all(&log_file).await; // create dir + if let Err(e) = fs_err::tokio::create_dir_all(&log_file).await { + tracing::warn!("failed to create log directory {log_file:?}: {e}"); + } log_file.push(file); - Ok(log_file) + log_file } #[tracing::instrument(skip(self), fields(%drv), err)] pub async fn new_log_file( &self, - drv: &nix_utils::StorePath, - ) -> anyhow::Result { - let log_file = self.construct_log_file_path(drv).await?; + drv: &StorePath, + ) -> Result { + let log_file = self.construct_log_file_path(drv).await; tracing::debug!("opening {log_file:?}"); - Ok(fs_err::tokio::File::options() + fs_err::tokio::File::options() .create(true) .truncate(true) .write(true) .read(false) .mode(0o666) .open(log_file) - .await?) + .await } - #[tracing::instrument(skip(self, new_ids, new_builds_by_id, new_builds_by_path))] + #[allow(clippy::cast_possible_truncation)] + #[tracing::instrument(skip(self, new_builds_by_id, new_builds_by_path, finished_drvs), err)] + async fn process_single_build( + &self, + id: BuildID, + new_builds_by_id: Arc>>>, + new_builds_by_path: Arc>>, + finished_drvs: Arc>>, + ) -> Result, StateError> { + let Some(build) = new_builds_by_id.read().get(&id).cloned() else { + return Ok(None); + }; + + let new_runnable = Arc::new(parking_lot::RwLock::new(HashSet::>::new())); + let nr_added: Arc = Arc::new(0.into()); + let now = Instant::now(); + + self.create_build( + build, + nr_added.clone(), + new_builds_by_id, + new_builds_by_path, + finished_drvs, + new_runnable.clone(), + ) + .await?; + + // we should never run into this issue + #[allow(clippy::cast_possible_truncation)] + let elapsed = now.elapsed().as_millis() as u64; + + Ok(Some(ProcessedBuild { + _id: id, + nr_added, + new_runnable, + elapsed, + })) + } + + #[tracing::instrument(skip(self, new_ids, new_builds_by_id, new_builds_by_path), err)] async fn process_new_builds( &self, new_ids: Vec, new_builds_by_id: Arc>>>, - new_builds_by_path: HashMap>, - ) { - let finished_drvs = Arc::new(parking_lot::RwLock::new( - HashSet::::new(), - )); + new_builds_by_path: HashMap>, + ) -> Result { + use futures::stream::StreamExt as _; + let finished_drvs = Arc::new(parking_lot::RwLock::new(HashSet::::new())); + let new_builds_by_path = Arc::new(new_builds_by_path); let starttime = jiff::Timestamp::now(); - for id in new_ids { - let Some(build) = new_builds_by_id.read().get(&id).cloned() else { - continue; - }; - let new_runnable = Arc::new(parking_lot::RwLock::new(HashSet::>::new())); - let nr_added: Arc = Arc::new(0.into()); - let now = Instant::now(); + let mut futures = futures::stream::FuturesUnordered::new(); + let mut ids_iter = new_ids.into_iter(); + + for _ in 0..MAX_CONCURRENT_BUILD_INJECTION { + if let Some(id) = ids_iter.next() { + futures.push(Box::pin(Self::process_single_build( + self, + id, + new_builds_by_id.clone(), + new_builds_by_path.clone(), + finished_drvs.clone(), + ))); + } + } - Box::pin(self.create_build( - build, - nr_added.clone(), - new_builds_by_id.clone(), - &new_builds_by_path, - finished_drvs.clone(), - new_runnable.clone(), - )) - .await; + let mut early_exit = false; - // we should never run into this issue - #[allow(clippy::cast_possible_truncation)] - self.metrics - .build_read_time_ms - .inc_by(now.elapsed().as_millis() as u64); + while let Some(result) = futures.next().await { + let Some(ProcessedBuild { + nr_added, + new_runnable, + elapsed, + .. + }) = result? + else { + continue; + }; + + self.metrics.build_read_time_ms.inc_by(elapsed); { let new_runnable = new_runnable.read(); @@ -669,13 +915,28 @@ impl State { if let Ok(added_u64) = u64::try_from(nr_added.load(Ordering::Relaxed)) { self.metrics.nr_builds_read.inc_by(added_u64); } - let stop_queue_run_after = self.config.get_stop_queue_run_after(); - if let Some(stop_queue_run_after) = stop_queue_run_after - && jiff::Timestamp::now() > (starttime + stop_queue_run_after) - { - self.metrics.queue_checks_early_exits.inc(); - break; + // Check early exit only once, after each completed build. + // If already exiting, we let in-flight tasks drain naturally. + if !early_exit { + let stop_queue_run_after = self.config.get_stop_queue_run_after(); + if let Some(stop_queue_run_after) = stop_queue_run_after + && jiff::Timestamp::now() > (starttime + stop_queue_run_after) + { + early_exit = true; + self.metrics.queue_checks_early_exits.inc(); + } + } + + // Queue the next build if not exiting early. + if !early_exit && let Some(id) = ids_iter.next() { + futures.push(Box::pin(Self::process_single_build( + self, + id, + new_builds_by_id.clone(), + new_builds_by_path.clone(), + finished_drvs.clone(), + ))); } } @@ -690,10 +951,11 @@ impl State { if let Some(fod_checker) = &self.fod_checker { fod_checker.trigger_traverse(); } + Ok(early_exit) } #[tracing::instrument(skip(self), err)] - async fn process_queue_change(&self) -> anyhow::Result<()> { + async fn process_queue_change(&self) -> Result<(), db::Error> { let mut db = self.db.get().await?; let curr_ids: HashMap<_, _> = db .get_not_finished_builds_fast() @@ -721,22 +983,32 @@ impl State { } Ok(()) } +} + +/// Errors from looking up derivations. +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub enum DrvLookupError { + #[error("drv not found")] + DrvNotFound, + #[error("derivation not found")] + DerivationNotFound, +} + +impl State { #[tracing::instrument(skip(self), fields(%drv_path))] pub async fn queue_one_build( &self, jobset_id: i32, - drv_path: &nix_utils::StorePath, - ) -> anyhow::Result<()> { + drv_path: &StorePath, + ) -> Result<(), StateError> { let mut db = self.db.get().await?; - let drv = nix_utils::query_drv(&self.store, drv_path) - .await? - .ok_or_else(|| anyhow::anyhow!("drv not found"))?; + let drv = self.read_derivation(drv_path).await?; db.insert_debug_build( - self.store.store_dir(), + self.pool.store_dir(), jobset_id, drv_path, - std::str::from_utf8(&drv.platform).expect("platform must be valid UTF-8"), + std::str::from_utf8(&drv.platform).map_err(StateError::InvalidPlatformUtf8)?, ) .await?; @@ -747,15 +1019,18 @@ impl State { } #[tracing::instrument(skip(self), err)] - pub(crate) async fn manually_add_queue_build(&self, build_id: BuildID) -> anyhow::Result<()> { + pub(crate) async fn manually_add_queue_build( + &self, + build_id: BuildID, + ) -> Result<(), StateError> { let mut new_ids = Vec::::new(); let mut new_builds_by_id = HashMap::>::new(); - let mut new_builds_by_path = HashMap::>::new(); + let mut new_builds_by_path = HashMap::>::new(); { let mut conn = self.db.get().await?; for b in conn - .get_not_finished_builds(self.store.store_dir()) + .get_not_finished_builds(self.pool.store_dir()) .await? .into_iter() .filter(|b| b.id == build_id) @@ -782,22 +1057,23 @@ impl State { } let new_builds_by_id = Arc::new(parking_lot::RwLock::new(new_builds_by_id)); - Box::pin(self.process_new_builds(new_ids, new_builds_by_id, new_builds_by_path)).await; + let _early_exit = + Box::pin(self.process_new_builds(new_ids, new_builds_by_id, new_builds_by_path)) + .await?; Ok(()) } #[tracing::instrument(skip(self), err)] - pub async fn get_queued_builds(&self) -> anyhow::Result<()> { + pub async fn get_queued_builds(&self) -> Result { self.metrics.queue_checks_started.inc(); let mut new_ids = Vec::::with_capacity(1000); let mut new_builds_by_id = HashMap::>::with_capacity(1000); - let mut new_builds_by_path = - HashMap::>::with_capacity(1000); + let mut new_builds_by_path = HashMap::>::with_capacity(1000); { let mut conn = self.db.get().await?; - for b in conn.get_not_finished_builds(self.store.store_dir()).await? { + for b in conn.get_not_finished_builds(self.pool.store_dir()).await? { let jobset = self .jobsets .create(&mut conn, b.jobset_id, &b.project, &b.jobset) @@ -816,8 +1092,10 @@ impl State { tracing::debug!("new_builds_by_path: {new_builds_by_path:?}"); let new_builds_by_id = Arc::new(parking_lot::RwLock::new(new_builds_by_id)); - Box::pin(self.process_new_builds(new_ids, new_builds_by_id, new_builds_by_path)).await; - Ok(()) + let early_exit = + Box::pin(self.process_new_builds(new_ids, new_builds_by_id, new_builds_by_path)) + .await?; + Ok(early_exit) } #[tracing::instrument(skip(self))] @@ -833,7 +1111,7 @@ impl State { } #[tracing::instrument(skip(self), err)] - async fn queue_monitor_loop(&self) -> anyhow::Result<()> { + async fn queue_monitor_loop(&self) -> Result<(), StateError> { let mut listener = self .db .listener(vec![ @@ -848,11 +1126,14 @@ impl State { loop { let before_work = Instant::now(); - self.store.clear_path_info_cache(); - if let Err(e) = self.get_queued_builds().await { - tracing::error!("get_queue_builds failed inside queue monitor loop: {e}"); - continue; - } + // no cache in daemon protocol + let early_exit = match self.get_queued_builds().await { + Ok(early_exit) => early_exit, + Err(e) => { + tracing::error!("get_queue_builds failed inside queue monitor loop: {e}"); + continue; + } + }; #[allow(clippy::cast_possible_truncation)] self.metrics @@ -861,7 +1142,21 @@ impl State { let before_sleep = Instant::now(); let queue_trigger_timer = self.config.get_queue_trigger_timer(); - let notification = if let Some(timer) = queue_trigger_timer { + let notification = if early_exit { + // Short poll: process maybe pending notifications, then re-run immediately. + let short_poll = std::time::Duration::from_millis(100); + tokio::select! { + () = tokio::time::sleep(short_poll) => {"timer_reached".into()}, + v = listener.try_next() => match v { + Ok(Some(v)) => v.channel().to_owned(), + Ok(None) => continue, + Err(e) => { + tracing::warn!("PgListener failed with e={e}"); + continue; + } + }, + } + } else if let Some(timer) = queue_trigger_timer { tokio::select! { () = tokio::time::sleep(timer) => {"timer_reached".into()}, v = listener.try_next() => match v { @@ -970,13 +1265,13 @@ impl State { let task = tokio::task::spawn({ async move { loop { - let local_store = nix_utils::LocalStore::init(); + let local_store = self.pool.clone(); let s3_stores: Vec = { let r = self.remote_stores.read(); r.iter() .filter_map(|s| match s { RemoteStoreBackend::S3(s) => Some(s.clone()), - _ => None, + RemoteStoreBackend::NixCopy(_) => None, }) .collect() }; @@ -1060,7 +1355,7 @@ impl State { build_id: uuid::Uuid, machine_id: uuid::Uuid, step_status: db::models::StepStatus, - ) -> anyhow::Result<()> { + ) -> Result<(), db::Error> { let build_id_and_step_nr = self.machines.get_machine_by_id(machine_id).and_then(|m| { tracing::debug!( "get job from machine by build_id: build_id={build_id} m={}", @@ -1086,21 +1381,33 @@ impl State { .await?; Ok(()) } +} + +/// Errors from looking up steps/jobs in the in-memory queues. +#[derive(Debug, thiserror::Error)] +pub enum StepLookupError { + #[error("step is missing in queues.scheduled")] + StepNotScheduled, + + #[error("job is missing in machine.jobs m={0}")] + JobNotOnMachine(String), +} +impl State { #[allow(clippy::too_many_lines)] #[tracing::instrument(skip(self, output), fields(%machine_id, %drv_path), err)] pub async fn succeed_step( &self, machine_id: uuid::Uuid, - drv_path: &nix_utils::StorePath, + drv_path: &StorePath, output: BuildOutput, - ) -> anyhow::Result<()> { + ) -> Result<(), StateError> { tracing::info!("marking job as done: drv_path={drv_path}"); let item = self .queues .remove_job_from_scheduled(drv_path) .await - .ok_or_else(|| anyhow::anyhow!("Step is missing in queues.scheduled"))?; + .ok_or(StateError::from(StepLookupError::StepNotScheduled))?; item.step_info.step.set_finished(true); @@ -1110,19 +1417,20 @@ impl State { // the queue runner and builder are disagreeing on what the drv itself // means (regardless of what it produces) which would be an "all bets // are off" bug. - if let Some(expected) = item.step_info.step.get_output_paths(self.store.store_dir()) { + if let Some(expected) = item.step_info.step.get_output_paths(self.pool.store_dir()) { for (name, expected_path) in &expected { let Some(expected_path) = expected_path else { continue; // path not statically known (Deferred/CAFloating/Impure) }; - if let Some(actual_path) = output.outputs.get(name) { - anyhow::ensure!( - expected_path == actual_path, - "output path mismatch for output `{name}` of {drv_path}: \ - expected {}, got {}", - self.store.print_store_path(expected_path), - self.store.print_store_path(actual_path), - ); + if let Some(actual_path) = output.outputs.get(name) + && expected_path != actual_path + { + return Err(StateError::from(ResolutionError::OutputPathMismatch { + name: name.to_string(), + drv: drv_path.clone(), + expected: self.pool.store_dir().display(expected_path).to_string(), + actual: self.pool.store_dir().display(actual_path).to_string(), + })); } } } @@ -1131,10 +1439,9 @@ impl State { "removing job from machine: drv_path={drv_path} m={}", item.machine.id ); - let mut job = item - .machine - .remove_job(drv_path) - .ok_or_else(|| anyhow::anyhow!("Job is missing in machine.jobs m={}", item.machine,))?; + let mut job = item.machine.remove_job(drv_path).ok_or_else(|| { + StateError::from(StepLookupError::JobNotOnMachine(item.machine.to_string())) + })?; self.queues .remove_job(&item.step_info, &item.build_queue) .await; @@ -1152,7 +1459,7 @@ impl State { finish_build_step( &self.db, - &self.store, + self.pool.store_dir(), job.build_id, job.step_nr, &job.result, @@ -1161,34 +1468,50 @@ impl State { ) .await?; - // Copy outputs to non-S3 (FFI) stores so that - // hydra-notify plugins (e.g. DeclarativeJobsets) can read them. + // Copy outputs to non-S3 stores via `nix copy`. { - let ffi_base_stores: Vec<(String, nix_utils::BaseStoreImpl)> = { + let nix_copy_uris: Vec = { let stores = self.remote_stores.read(); stores .iter() .filter_map(|s| match s { - RemoteStoreBackend::Nix(s) => { - Some((s.uri.clone(), s.as_base_store().clone())) - } - _ => None, + RemoteStoreBackend::NixCopy(uri) => Some(uri.clone()), + RemoteStoreBackend::S3(_) => None, }) .collect() }; - let outputs_to_copy = output.outputs.values().cloned().collect::>(); - for (uri, base_store) in &ffi_base_stores { - if let Err(e) = nix_utils::copy_paths( - self.store.as_base_store(), - base_store, - &outputs_to_copy, - false, - false, - false, - ) - .await - { - tracing::error!("Failed to copy outputs to store {uri}: {e}"); + if !nix_copy_uris.is_empty() { + let paths: Vec = output + .outputs + .values() + .map(|p| self.pool.store_dir().display(p).to_string()) + .collect(); + for dest_uri in &nix_copy_uris { + let output = tokio::process::Command::new("nix") + .arg("--extra-experimental-features") + .arg("nix-command") + .arg("copy") + .arg("--from") + .arg(self.nix_daemon_config.to_uri()) + .arg("--to") + .arg(dest_uri) + .args(&paths) + .output() + .await; + match output { + Ok(out) if out.status.success() => { + tracing::info!("Copied {} paths to {dest_uri}", paths.len()); + } + Ok(out) => { + tracing::error!( + "nix copy to {dest_uri} failed: {}", + str::from_utf8(&out.stderr).unwrap_or("Invalid UTF-8") + ); + } + Err(e) => { + tracing::error!("Failed to run nix copy to {dest_uri}: {e}"); + } + } } } } @@ -1225,31 +1548,31 @@ impl State { // `Realisations` table (for non-S3 / FFI stores) once nix is // updated to 2.35, which uses path-based DrvOutput matching // the harmonia types. - if let Some(drv_guard) = item.step_info.step.get_drv() { - let drv_ref = drv_guard.as_ref().unwrap(); + let drv_guard = item.step_info.step.get_drv(); + if let Some(drv_ref) = drv_guard.as_ref() { let has_ca_floating = drv_ref .outputs .values() - .any(|o| matches!(o, nix_utils::DerivationOutput::CAFloating(_))); + .any(|o| matches!(o, DerivationOutput::CAFloating(_))); if has_ca_floating { let s3_stores: Vec = { let r = self.remote_stores.read(); r.iter() .filter_map(|s| match s { RemoteStoreBackend::S3(s) => Some(s.clone()), - _ => None, + RemoteStoreBackend::NixCopy(_) => None, }) .collect() }; for (output_name, out_path) in &output.outputs { - let realisation = nix_utils::Realisation { - key: nix_utils::DrvOutput { + let realisation = Realisation { + key: DrvOutput { drv_path: drv_path.clone(), output_name: output_name.clone(), }, - value: nix_utils::UnkeyedRealisation { + value: UnkeyedRealisation { out_path: out_path.clone(), - signatures: Default::default(), + signatures: BTreeSet::default(), }, }; for s3 in &s3_stores { @@ -1276,11 +1599,11 @@ impl State { for b in &direct { let is_cached = job.build_id != b.id || job.result.is_cached; tx.mark_succeeded_build( - get_mark_build_sccuess_data(&self.store, b, &output), + get_mark_build_sccuess_data(b, &output), is_cached, start_time, stop_time, - self.store.store_dir(), + self.pool.store_dir(), ) .await?; self.metrics.nr_builds_done.inc(); @@ -1315,15 +1638,16 @@ impl State { }; let resolved_drv = output.outputs.get(&output_name).cloned().ok_or_else(|| { - anyhow::anyhow!( - "Dynamic rdep references output `{output_name}` not produced by {drv_path}" - ) + StateError::from(ResolutionError::DynRdepOutputMissing { + output: output_name.to_string(), + drv: drv_path.clone(), + }) })?; // Find a build associated with this step. For intermediate steps // (not top-level), `direct` is empty, so we walk the dependency // chain via `get_dependents` to find the owning build. - let build = if let Some(b) = direct.get(0) { + let build = if let Some(b) = direct.first() { b.clone() } else { let mut dependents = HashSet::new(); @@ -1342,15 +1666,15 @@ impl State { // finished_drvs is not necessary as it is only a memoization table to reduce // checks if a dependency is finished from the database. // new_steps is not necessary either as - let new_runnable: Arc>>> = Default::default(); + let new_runnable: Arc>>> = Arc::default(); let new_step = match self .create_step( build.clone(), resolved_drv.clone(), None, Some((dependent_step.clone(), relation)), - Default::default(), - Default::default(), + Arc::default(), + Arc::default(), new_runnable.clone(), ) .await @@ -1386,16 +1710,16 @@ impl State { pub async fn fail_step( &self, machine_id: uuid::Uuid, - drv_path: &nix_utils::StorePath, + drv_path: &StorePath, state: BuildResultState, timings: BuildTimings, - ) -> anyhow::Result<()> { + ) -> Result<(), StateError> { tracing::info!("removing job from running in system queue: drv_path={drv_path}"); let item = self .queues .remove_job_from_scheduled(drv_path) .await - .ok_or_else(|| anyhow::anyhow!("Step is missing in queues.scheduled"))?; + .ok_or(StateError::from(StepLookupError::StepNotScheduled))?; item.step_info.step.set_finished(false); @@ -1403,14 +1727,13 @@ impl State { "removing job from machine: drv_path={drv_path} m={}", item.machine.id ); - let mut job = item - .machine - .remove_job(drv_path) - .ok_or_else(|| anyhow::anyhow!("Job is missing in machine.jobs m={}", item.machine))?; + let mut job = item.machine.remove_job(drv_path).ok_or_else(|| { + StateError::from(StepLookupError::JobNotOnMachine(item.machine.to_string())) + })?; job.result.step_status = BuildStatus::Failed; // this can override step_status to something more specific - job.result.update_with_result_state(&state); + job.result.update_with_result_state(state); job.result.set_stop_time_now(); job.result.set_overhead(timings.get_overhead())?; @@ -1450,7 +1773,7 @@ impl State { finish_build_step( &self.db, - &self.store, + self.pool.store_dir(), job.build_id, job.step_nr, &job.result, @@ -1476,21 +1799,33 @@ impl State { ) .await } +} + +/// Errors from looking up machines/jobs by UUID. +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub enum MachineLookupError { + #[error("machine with machine_id not found")] + MachineNotFound, + #[error("job with build_id not found")] + JobNotFound, +} + +impl State { #[tracing::instrument(skip(self, output), fields(%machine_id, build_id=%build_id), err)] pub async fn succeed_step_by_uuid( &self, build_id: uuid::Uuid, machine_id: uuid::Uuid, output: BuildOutput, - ) -> anyhow::Result<()> { + ) -> Result<(), StateError> { let machine = self .machines .get_machine_by_id(machine_id) - .ok_or_else(|| anyhow::anyhow!("Machine with machine_id not found"))?; + .ok_or(StateError::from(MachineLookupError::MachineNotFound))?; let drv_path = machine .get_job_drv_for_build_id(build_id) - .ok_or_else(|| anyhow::anyhow!("Job with build_id not found"))?; + .ok_or(StateError::from(MachineLookupError::JobNotFound))?; self.succeed_step(machine_id, &drv_path, output).await } @@ -1502,14 +1837,14 @@ impl State { machine_id: uuid::Uuid, state: BuildResultState, timings: BuildTimings, - ) -> anyhow::Result<()> { + ) -> Result<(), StateError> { let machine = self .machines .get_machine_by_id(machine_id) - .ok_or_else(|| anyhow::anyhow!("Machine with machine_id not found"))?; + .ok_or(StateError::from(MachineLookupError::MachineNotFound))?; let drv_path = machine .get_job_drv_for_build_id(build_id) - .ok_or_else(|| anyhow::anyhow!("Job with build_id not found"))?; + .ok_or(StateError::from(MachineLookupError::JobNotFound))?; self.fail_step(machine_id, &drv_path, state, timings).await } @@ -1518,11 +1853,11 @@ impl State { #[tracing::instrument(skip(self, machine, job, step), fields(%drv_path), err)] async fn inner_fail_job( &self, - drv_path: &nix_utils::StorePath, + drv_path: &StorePath, machine: Option>, mut job: machine::Job, step: Arc, - ) -> anyhow::Result<()> { + ) -> Result<(), StateError> { if !job.result.has_stop_time() { job.result.set_stop_time_now(); } @@ -1530,7 +1865,7 @@ impl State { if job.step_nr != 0 { finish_build_step( &self.db, - &self.store, + self.pool.store_dir(), job.build_id, job.step_nr, &job.result, @@ -1566,7 +1901,7 @@ impl State { } tx.create_build_step( - self.store.store_dir(), + self.pool.store_dir(), None, b.id, step.get_drv_path(), @@ -1582,7 +1917,7 @@ impl State { } else { Some(job.build_id) }, - step.get_output_paths(self.store.store_dir()) + step.get_output_paths(self.pool.store_dir()) .unwrap_or_default() .into_iter() .collect(), @@ -1619,12 +1954,11 @@ impl State { // Remember failed paths in the database so that they won't be built again. if job.result.step_status != BuildStatus::CachedFailure && job.result.can_cache { for (_, path) in step - .get_output_paths(self.store.store_dir()) + .get_output_paths(self.pool.store_dir()) .unwrap_or_default() { if let Some(path) = path { - tx.insert_failed_paths(self.store.store_dir(), &path) - .await?; + tx.insert_failed_paths(self.pool.store_dir(), &path).await?; } } } @@ -1681,7 +2015,7 @@ impl State { &self, build: Arc, step: Arc, - ) -> anyhow::Result<()> { + ) -> Result<(), StateError> { // Some step previously failed, so mark the build as failed right away. tracing::warn!( "marking build {} as cached failure due to '{}'", @@ -1699,7 +2033,7 @@ impl State { // Find the previous build step record, first by derivation path, then by output // path. let mut propagated_from = tx - .get_last_build_step_id(self.store.store_dir(), step.get_drv_path()) + .get_last_build_step_id(self.pool.store_dir(), step.get_drv_path()) .await? .unwrap_or_default(); @@ -1708,15 +2042,15 @@ impl State { // PreviousFailure is returned, so this should never yield None let outputs = step - .get_output_paths(self.store.store_dir()) + .get_output_paths(self.pool.store_dir()) .unwrap_or_default(); for (name, path) in &outputs { let res = if let Some(path) = path { - tx.get_last_build_step_id_for_output_path(self.store.store_dir(), path) + tx.get_last_build_step_id_for_output_path(self.pool.store_dir(), path) .await } else { tx.get_last_build_step_id_for_output_with_drv( - self.store.store_dir(), + self.pool.store_dir(), step.get_drv_path(), name.as_ref(), ) @@ -1730,7 +2064,7 @@ impl State { } tx.create_build_step( - self.store.store_dir(), + self.pool.store_dir(), None, build.id, step.get_drv_path(), @@ -1739,7 +2073,7 @@ impl State { BuildStatus::CachedFailure, None, Some(propagated_from), - step.get_output_paths(self.store.store_dir()) + step.get_output_paths(self.pool.store_dir()) .unwrap_or_default() .into_iter() .collect(), @@ -1778,10 +2112,10 @@ impl State { build: Arc, nr_added: Arc, new_builds_by_id: Arc>>>, - new_builds_by_path: &HashMap>, - finished_drvs: Arc>>, + new_builds_by_path: Arc>>, + finished_drvs: Arc>>, new_runnable: Arc>>>, - ) { + ) -> Result<(), StateError> { self.metrics.queue_build_loads.inc(); tracing::info!("loading build {} ({})", build.id, build.full_job_name()); nr_added.fetch_add(1, Ordering::Relaxed); @@ -1790,11 +2124,11 @@ impl State { new_builds_by_id.remove(&build.id); } - if !self.store.is_valid_path(&build.drv_path).await { + if !daemon_client_utils::is_valid_path(&self.pool, &build.drv_path).await? { tracing::error!( "aborting GC'ed build id={} path={}", build.id, - self.store.print_store_path(&build.drv_path) + self.pool.store_dir().display(&build.drv_path) ); if !build.get_finished_in_db() { match self.db.get().await { @@ -1813,7 +2147,7 @@ impl State { build.set_finished_in_db(true); self.metrics.nr_builds_done.inc(); - return; + return Ok(()); } // Create steps for this derivation and its dependencies. @@ -1837,7 +2171,7 @@ impl State { if let Err(e) = self.handle_previous_failure(build, step).await { tracing::error!("Failed to handle previous failure: {e}"); } - return; + return Ok(()); } }; @@ -1855,6 +2189,7 @@ impl State { let mut stream = futures::StreamExt::map(tokio_stream::iter(builds), |b| { let nr_added = nr_added.clone(); let new_builds_by_id = new_builds_by_id.clone(); + let new_builds_by_path = new_builds_by_path.clone(); let finished_drvs = finished_drvs.clone(); let new_runnable = new_runnable.clone(); async move { @@ -1862,7 +2197,7 @@ impl State { if let Some(j) = new_builds_by_id.read().get(&b) { j.clone() } else { - return; + return Ok(()); } }; @@ -1874,11 +2209,13 @@ impl State { finished_drvs, new_runnable, )) - .await; + .await } }) .buffered(10); - while tokio_stream::StreamExt::next(&mut stream).await.is_some() {} + while let Some(result) = tokio_stream::StreamExt::next(&mut stream).await { + result?; + } } if let Some(step) = step { @@ -1902,6 +2239,7 @@ impl State { tracing::error!("failed to handle cached build: {e}"); } } + Ok(()) } #[allow(clippy::too_many_lines, clippy::too_many_arguments)] @@ -1917,10 +2255,10 @@ impl State { async fn create_step( &self, build: Arc, - drv_path: nix_utils::StorePath, + drv_path: StorePath, referring_build: Option>, referring_step: Option<(Arc, drv::OutputNameChain)>, - finished_drvs: Arc>>, + finished_drvs: Arc>>, new_steps: Arc>>>, new_runnable: Arc>>>, ) -> CreateStepResult { @@ -1941,16 +2279,59 @@ impl State { .map(|(step, relation)| (step, relation.clone())), ); if !is_new { + // Re-check whether the step's outputs have appeared in the store + // since it was first created. This handles the case where outputs + // became available between poll cycles (e.g. built by a concurrent + // step, substituted, or uploaded externally). Without this check, + // builds whose outputs are now cached get stuck in an infinite + // re-load loop: the DB says finished=0, the step already exists in + // memory, and create_build never reaches handle_cached_build. + // + // To be clear, builds that go through gRPC do not need this. The + // builder will push the info to the queue runner so there is no + // polling race condition. It is likely that this case happened + // because IFD in the evaluator was causing builds on the host, and + // *those* were subject to the race condition --- build-relevant + // store objects shouldn't be unexpected appearing in the host store + // otherwise. + // + // TODO once we properly feed IFD builds in to Hydra to be + // distributed, remove this hack. + if step.get_finished() { + return CreateStepResult::None; + } + if let Some(output_paths) = step.get_output_paths(self.pool.store_dir()) { + // All output paths must be known (Some) and valid in + // the store for the step to count as finished. CA + // floating outputs have None paths until built. + let all_resolved = output_paths.values().all(Option::is_some); + let all_valid = if all_resolved { + let mut valid = true; + for path in output_paths.values().flatten() { + if !daemon_client_utils::is_valid_path(&self.pool, path) + .await + .unwrap_or(false) + { + valid = false; + break; + } + } + valid + } else { + false + }; + if all_valid { + finished_drvs.write().insert(drv_path.clone()); + step.set_finished(true); + return CreateStepResult::None; + } + } return CreateStepResult::Valid(step); } self.metrics.queue_steps_created.inc(); tracing::debug!("considering derivation '{drv_path}'"); - let Some(drv) = nix_utils::query_drv(&self.store, &drv_path) - .await - .ok() - .flatten() - else { + let Some(drv) = self.read_derivation(&drv_path).await.ok() else { tracing::warn!("create_step: could not query derivation {drv_path}, skipping"); return CreateStepResult::None; }; @@ -1958,14 +2339,15 @@ impl State { fod_checker.add_ca_drv_parsed(&drv_path, &drv); } - let system_type = std::str::from_utf8(&drv.platform).expect("platform must be valid UTF-8"); - #[allow(clippy::cast_precision_loss)] - self.metrics.observe_build_input_drvs( - harmonia_store_core::derivation::DerivationInputs::from(&drv.inputs) - .drvs - .len() as f64, - system_type, - ); + if let Ok(system_type) = std::str::from_utf8(&drv.platform) { + #[allow(clippy::cast_precision_loss)] + self.metrics.observe_build_input_drvs( + harmonia_store_derivation::derivation::DerivationInputs::from(&drv.inputs) + .drvs + .len() as f64, + system_type, + ); + } let use_substitutes = self.config.get_use_substitutes(); // TODO: check all remote stores @@ -1973,10 +2355,22 @@ impl State { let r = self.remote_stores.read(); r.iter().find_map(|s| match s { RemoteStoreBackend::S3(s) => Some(s.clone()), - _ => None, + RemoteStoreBackend::NixCopy(_) => None, }) }; - let output_paths = nix_utils::output_paths(&drv, self.store.store_dir()); + let output_paths: BTreeMap> = drv + .outputs + .iter() + .map(|(name, output)| { + ( + name.clone(), + output + .path(self.pool.store_dir(), &drv.name, name) + .ok() + .flatten(), + ) + }) + .collect(); let known_outputs = self .query_known_drv_outputs(&drv_path) .await @@ -1984,7 +2378,28 @@ impl State { tracing::warn!("Could not query known outputs, continuing: {e}"); BTreeMap::new() }); - let missing_local_outputs = self.store.query_missing_outputs(output_paths.clone()).await; + // Outputs with None paths (CA floating) are always missing — + // their path is unknown until the derivation is built. + let missing_local_outputs: BTreeMap> = { + let mut missing = BTreeMap::new(); + for (name, path) in &output_paths { + match path { + Some(path) => { + if !daemon_client_utils::is_valid_path(&self.pool, path) + .await + .unwrap_or(false) + { + missing.insert(name.clone(), Some(path.clone())); + } + } + None => { + // CA floating: output path unknown, definitely missing + missing.insert(name.clone(), None); + } + } + } + missing + }; // Handle paths that aren't in the database (for resolution) // existing_local_outputs = output_paths - missing_local_outputs // unregistered_local_outputs = existing_local_outputs - known_outputs @@ -2000,7 +2415,7 @@ impl State { if !unregistered_local_outputs.is_empty() && let Err(e) = crate::utils::make_local_step( &self.db, - &self.store, + self.pool.store_dir(), build.id, &drv_path, &unregistered_local_outputs, @@ -2017,22 +2432,22 @@ impl State { .await; if !missing.is_empty() && missing_local_outputs.is_empty() { // we have all paths locally, so we can just upload them to the remote_store - if let Ok(log_file) = self.construct_log_file_path(&drv_path).await { - let missing_paths: Vec = + { + let log_file = self.construct_log_file_path(&drv_path).await; + let missing_paths: Vec = missing.values().filter_map(Clone::clone).collect(); self.uploader - .schedule_upload( - missing_paths, - format!("log/{drv_path}"), - log_file.to_string_lossy().to_string(), - ) + .schedule_upload(missing_paths, format!("log/{drv_path}"), log_file) .await; missing.clear(); } } missing } else { - self.store.query_missing_outputs(output_paths).await + // Without a remote store, just check the local store. + // Reuse missing_local_outputs which already includes None + // (CA floating) paths as missing. + missing_local_outputs.clone() }; let input_drvs = drv::input_drvs(&drv); @@ -2053,7 +2468,7 @@ impl State { self.metrics.nr_substitutes_started.inc(); crate::utils::substitute_output( self.db.clone(), - nix_utils::LocalStore::init(), + self.pool.clone(), o, build.id, &drv_path, @@ -2078,6 +2493,7 @@ impl State { } substituted == missing_outputs_len } else { + // CA floating outputs have None paths — they must be built. missing_outputs.is_empty() }; @@ -2151,18 +2567,17 @@ impl State { #[tracing::instrument(skip(self))] async fn query_known_drv_outputs( &self, - drv_path: &nix_utils::StorePath, - ) -> anyhow::Result> { + drv_path: &StorePath, + ) -> Result, db::Error> { let mut db = self.db.get().await?; let mut tx = db.begin_transaction().await?; - Ok(tx - .find_build_step_outputs(self.store.store_dir(), drv_path) - .await?) + tx.find_build_step_outputs(self.pool.store_dir(), drv_path) + .await } #[tracing::instrument(skip(self, step), ret, level = "debug")] async fn check_cached_failure(&self, step: Arc) -> bool { - let Some(drv_outputs) = step.get_output_paths(self.store.store_dir()) else { + let Some(drv_outputs) = step.get_output_paths(self.pool.store_dir()) else { return false; }; @@ -2171,18 +2586,15 @@ impl State { }; conn.check_if_paths_failed( - self.store.store_dir(), - &drv_outputs - .iter() - .filter_map(|(_, path)| path.clone()) - .collect::>(), + self.pool.store_dir(), + &drv_outputs.values().flatten().cloned().collect::>(), ) .await .unwrap_or_default() } #[tracing::instrument(skip(self, build), fields(build_id=build.id), err)] - async fn handle_cached_build(&self, build: Arc) -> anyhow::Result<()> { + async fn handle_cached_build(&self, build: Arc) -> Result<(), StateError> { let res = self.get_build_output_cached(&build.drv_path).await?; { @@ -2192,11 +2604,11 @@ impl State { tracing::info!("marking build {} as succeeded (cached)", build.id); let now = jiff::Timestamp::now().as_second(); tx.mark_succeeded_build( - get_mark_build_sccuess_data(&self.store, &build, &res), + get_mark_build_sccuess_data(&build, &res), true, i32::try_from(now)?, // TODO i32::try_from(now)?, // TODO - self.store.store_dir(), + self.pool.store_dir(), ) .await?; self.metrics.nr_builds_done.inc(); @@ -2212,13 +2624,23 @@ impl State { #[tracing::instrument(skip(self), err)] async fn get_build_output_cached( &self, - drv_path: &nix_utils::StorePath, - ) -> anyhow::Result { - let drv = nix_utils::query_drv(&self.store, drv_path) - .await? - .ok_or_else(|| anyhow::anyhow!("Derivation not found"))?; + drv_path: &StorePath, + ) -> Result { + let drv = self.read_derivation(drv_path).await?; - let output_paths = nix_utils::output_paths(&drv, self.store.store_dir()); + let output_paths: BTreeMap> = drv + .outputs + .iter() + .map(|(name, output)| { + ( + name.clone(), + output + .path(self.pool.store_dir(), &drv.name, name) + .ok() + .flatten(), + ) + }) + .collect(); { let mut db = self.db.get().await?; for out_path in output_paths.values() { @@ -2226,48 +2648,49 @@ impl State { continue; }; let Some(db_build_output) = db - .get_build_output_for_path(self.store.store_dir(), out_path) + .get_build_output_for_path(self.pool.store_dir(), out_path) .await? else { continue; }; let build_id = db_build_output.id; - let Ok(mut res): anyhow::Result = db_build_output.try_into() else { + let Ok(mut res): Result = db_build_output.try_into() else { continue; }; res.products = db - .get_build_products_for_build_id(build_id, self.store.store_dir()) - .await? - .into_iter() - .map(build::BuildProduct::from_db) - .collect(); + .get_build_products_for_build_id(build_id, self.pool.store_dir()) + .await?; res.metrics = db .get_build_metrics_for_build_id(build_id) .await? .into_iter() - .map(Into::into) .collect(); return Ok(res); } } - let build_output = BuildOutput::new(&self.store, output_paths).await?; + let default_store: std::path::PathBuf = self.pool.store_dir().to_string().into(); + let real_dir = self.real_store_dir.as_deref().unwrap_or(&default_store); + let build_output = BuildOutput::new(&self.pool, real_dir, output_paths).await?; - #[allow(clippy::cast_precision_loss)] - self.metrics.observe_build_closure_size( - build_output.closure_size as f64, - std::str::from_utf8(&drv.platform).expect("platform must be valid UTF-8"), - ); + if let Ok(platform) = std::str::from_utf8(&drv.platform) { + #[allow(clippy::cast_precision_loss)] + self.metrics + .observe_build_closure_size(build_output.closure_size as f64, platform); + } Ok(build_output) } #[allow(unused)] - fn add_root(&self, store_path: &nix_utils::StorePath) { + fn add_root(&self, store_path: &StorePath) -> std::io::Result<()> { let roots_dir = self.config.get_roots_dir(); - nix_utils::add_root(&self.store, &roots_dir, store_path); + // Inline filesystem symlink for GC roots + let link_path = roots_dir.join(store_path.to_string()); + let store_path_full = format!("{}/{store_path}", self.pool.store_dir()); + fs_err::os::unix::fs::symlink(&store_path_full, &link_path) } async fn abort_unsupported(&self) { diff --git a/subprojects/hydra-queue-runner/src/state/queue.rs b/subprojects/hydra-queue-runner/src/state/queue.rs index ed5917249..4e53b4431 100644 --- a/subprojects/hydra-queue-runner/src/state/queue.rs +++ b/subprojects/hydra-queue-runner/src/state/queue.rs @@ -1,3 +1,4 @@ +use harmonia_store_path::StorePath; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; @@ -173,10 +174,10 @@ impl ScheduledItem { #[derive(Debug)] pub(super) struct InnerQueues { // flat list of all step infos in queues, owning those steps inner queue dont own them - jobs: HashMap>, + jobs: HashMap>, inner: HashMap>, #[allow(clippy::type_complexity)] - scheduled: parking_lot::RwLock>, + scheduled: parking_lot::RwLock>, } impl Default for InnerQueues { @@ -261,14 +262,14 @@ impl InnerQueues { } #[tracing::instrument(skip(self), fields(%drv))] - fn remove_job_from_scheduled(&self, drv: &nix_utils::StorePath) -> Option { + fn remove_job_from_scheduled(&self, drv: &StorePath) -> Option { let item = self.scheduled.write().remove(drv)?; item.step_info.set_already_scheduled(false); item.build_queue.decr_active(); Some(item) } - fn remove_job_by_path(&mut self, drv: &nix_utils::StorePath) { + fn remove_job_by_path(&mut self, drv: &StorePath) { if self.jobs.remove(drv).is_none() { tracing::error!("Failed to remove stepinfo drv={drv} from jobs!"); } @@ -287,7 +288,7 @@ impl InnerQueues { } #[tracing::instrument(skip(self))] - async fn kill_active_steps(&self) -> Vec<(nix_utils::StorePath, uuid::Uuid)> { + async fn kill_active_steps(&self) -> Vec<(StorePath, uuid::Uuid)> { tracing::info!("Kill all active steps"); let active = { let scheduled = self.scheduled.read(); @@ -453,7 +454,9 @@ impl Queues { metrics: &super::metrics::PromMetrics, ) -> i64 where - F: AsyncFn(JobConstraint) -> anyhow::Result, + F: AsyncFn( + JobConstraint, + ) -> Result, { let now = jiff::Timestamp::now(); let mut nr_steps_waiting_all_queues = 0; @@ -548,15 +551,12 @@ impl Queues { } #[tracing::instrument(skip(self), fields(%drv))] - pub async fn remove_job_from_scheduled( - &self, - drv: &nix_utils::StorePath, - ) -> Option { + pub async fn remove_job_from_scheduled(&self, drv: &StorePath) -> Option { let rq = self.inner.read().await; rq.remove_job_from_scheduled(drv) } - pub async fn remove_job_by_path(&self, drv: &nix_utils::StorePath) { + pub async fn remove_job_by_path(&self, drv: &StorePath) { let mut wq = self.inner.write().await; wq.remove_job_by_path(drv); } @@ -568,7 +568,7 @@ impl Queues { } #[tracing::instrument(skip(self))] - pub async fn kill_active_steps(&self) -> Vec<(nix_utils::StorePath, uuid::Uuid)> { + pub async fn kill_active_steps(&self) -> Vec<(StorePath, uuid::Uuid)> { let rq = self.inner.read().await; rq.kill_active_steps().await } diff --git a/subprojects/hydra-queue-runner/src/state/step.rs b/subprojects/hydra-queue-runner/src/state/step.rs index 55e16263e..182d49406 100644 --- a/subprojects/hydra-queue-runner/src/state/step.rs +++ b/subprojects/hydra-queue-runner/src/state/step.rs @@ -7,6 +7,9 @@ use hashbrown::{HashMap, HashSet}; use super::{Build, Jobset}; use db::models::BuildID; +use harmonia_store_derivation::derivation::Derivation; +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_path::{StoreDir, StorePath}; use super::drv::OutputNameChain; @@ -105,8 +108,8 @@ impl StepState { #[derive(Debug)] pub struct Step { - drv_path: nix_utils::StorePath, - drv: arc_swap::ArcSwapOption, + drv_path: StorePath, + drv: arc_swap::ArcSwapOption, runnable: AtomicBool, finished: AtomicBool, @@ -133,7 +136,7 @@ impl Hash for Step { impl Step { #[must_use] - pub fn new(drv_path: nix_utils::StorePath) -> Arc { + pub fn new(drv_path: StorePath) -> Arc { Arc::new(Self { drv_path, drv: arc_swap::ArcSwapOption::from(None), @@ -149,7 +152,7 @@ impl Step { } #[inline] - pub const fn get_drv_path(&self) -> &nix_utils::StorePath { + pub const fn get_drv_path(&self) -> &StorePath { &self.drv_path } @@ -178,18 +181,21 @@ impl Step { self.runnable.load(Ordering::SeqCst) } - pub fn get_drv(&self) -> Option>>> { - let drv = self.drv.load(); - if drv.is_some() { Some(drv) } else { None } + pub fn get_drv(&self) -> arc_swap::Guard>> { + self.drv.load() } - pub fn set_drv(&self, drv: nix_utils::Derivation) { + pub fn set_drv(&self, drv: Derivation) { self.drv.store(Some(Arc::new(drv))); } + /// # Panics + /// + /// Will panic if drv.platform is not a UTF-8 string pub fn get_system(&self) -> Option { let drv = self.drv.load_full(); drv.as_ref().map(|drv| { + #[allow(clippy::expect_used)] std::str::from_utf8(&drv.platform) .expect("platform must be valid UTF-8") .to_owned() @@ -220,11 +226,20 @@ impl Step { pub fn get_output_paths( &self, - store_dir: &nix_utils::StoreDir, - ) -> Option>> { + store_dir: &StoreDir, + ) -> Option>> { let drv = self.drv.load_full(); - drv.as_ref() - .map(|drv| nix_utils::output_paths(drv, store_dir)) + drv.as_ref().map(|drv| { + drv.outputs + .iter() + .map(|(name, output)| { + ( + name.clone(), + output.path(store_dir, &drv.name, name).ok().flatten(), + ) + }) + .collect() + }) } // TODO: properly parse derivation options instead of reading env vars directly @@ -399,7 +414,7 @@ impl Step { /// We collect into a `Vec` rather than returning an iterator because the /// write lock on the step's state must be released before the caller can /// do async work (e.g. `create_step`) with the results. - pub fn pop_dynamic_rdeps(&self) -> Vec<(Weak, nix_utils::OutputName, OutputNameChain)> { + pub fn pop_dynamic_rdeps(&self) -> Vec<(Weak, OutputName, OutputNameChain)> { let mut state = self.state.write(); state .rdeps @@ -463,7 +478,7 @@ impl Step { #[derive(Debug, Clone)] pub struct Steps { - inner: Arc>>>, + inner: Arc>>>, } impl Default for Steps { @@ -558,21 +573,18 @@ impl Steps { #[must_use] pub fn create( &self, - drv_path: &nix_utils::StorePath, + drv_path: &StorePath, referring_build: Option<&Arc>, referring_step: Option<(&Arc, OutputNameChain)>, ) -> (Arc, bool) { let mut is_new = false; let mut steps = self.inner.write(); let step = if let Some(step) = steps.get(drv_path) { - step.upgrade().map_or_else( - || { - steps.remove(drv_path); - is_new = true; - Step::new(drv_path.to_owned()) - }, - |step| step, - ) + step.upgrade().unwrap_or_else(|| { + steps.remove(drv_path); + is_new = true; + Step::new(drv_path.to_owned()) + }) } else { is_new = true; Step::new(drv_path.to_owned()) @@ -583,7 +595,7 @@ impl Steps { (step, is_new) } - pub fn remove(&self, drv_path: &nix_utils::StorePath) { + pub fn remove(&self, drv_path: &StorePath) { let mut steps = self.inner.write(); steps.remove(drv_path); } @@ -591,10 +603,12 @@ impl Steps { #[cfg(test)] mod tests { + #![allow(clippy::unwrap_used)] + use super::*; - fn drv(name: &str) -> nix_utils::StorePath { - nix_utils::parse_store_path(&format!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-{name}.drv")) + fn drv(name: &str) -> StorePath { + StorePath::from_base_path(&format!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-{name}.drv")).unwrap() } #[test] diff --git a/subprojects/hydra-queue-runner/src/state/step_info.rs b/subprojects/hydra-queue-runner/src/state/step_info.rs index 134bc4a89..bca8222c0 100644 --- a/subprojects/hydra-queue-runner/src/state/step_info.rs +++ b/subprojects/hydra-queue-runner/src/state/step_info.rs @@ -2,7 +2,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use db::models::BuildID; -use nix_utils::SingleDerivedPath; +use harmonia_store_derivation::derivation::{BasicDerivation, Derivation}; +use harmonia_store_derivation::derived_path::{OutputName, SingleDerivedPath}; +use harmonia_store_path::{StoreDir, StorePath}; use super::Step; use super::drv::flatten_chain; @@ -28,7 +30,7 @@ impl StepInfo { } /// Resolve a derivation's inputs into concrete store paths, returning a - /// [`BasicDerivation`](nix_utils::BasicDerivation). + /// [`BasicDerivation`](BasicDerivation). /// /// Returns [`None`] if the derivation is input-addressed (shouldn't be resolved), /// or if resolution fails because required outputs haven't been built yet. @@ -38,22 +40,12 @@ impl StepInfo { /// /// We only need a store dir, not a store, because all the info we need comes from the Hydra /// database. - pub(super) async fn try_resolve( - store_dir: &nix_utils::StoreDir, + pub(super) async fn try_resolve_force( + store_dir: &StoreDir, db: &db::Database, - drv: &nix_utils::Derivation, - resolved_drv_map: &hashbrown::HashMap, - ) -> Option { - // Input-addressed derivations should not be resolved because this would change their - // output paths. - let all_input_addressed = drv - .outputs - .values() - .any(|o| matches!(o, nix_utils::DerivationOutput::InputAddressed(_))); - if all_input_addressed { - return None; - } - + drv: &Derivation, + resolved_drv_map: &hashbrown::HashMap, + ) -> Option { // If there are no Built inputs, the derivation is already resolved. let has_built_inputs = drv .inputs @@ -74,12 +66,10 @@ impl StepInfo { let mut conn = db.get().await.ok()?; // Memoize depth-1 lookups across all chains resolved in this call. - let mut memo = std::collections::HashMap::< - (nix_utils::StorePath, nix_utils::OutputName), - Option, - >::new(); + let mut memo = + std::collections::HashMap::<(StorePath, OutputName), Option>::new(); - drv.try_resolve(store_dir, &mut |inputs| { + drv.try_resolve_force(store_dir, &mut |inputs| { tokio::task::block_in_place(|| { let rt = tokio::runtime::Handle::current(); @@ -102,20 +92,17 @@ impl StepInfo { .cloned() .unwrap_or_else(|| current.clone()); let key = (translated, output_name.clone()); - let result = match memo.get(&key) { - Some(cached) => cached.clone(), - None => { - let r = rt - .block_on( - conn.resolve_drv_output(store_dir, &key.0, &key.1), - ) - .unwrap_or_else(|e| { - tracing::warn!("resolve_drv_output failed: {e}"); - None - }); - memo.insert(key, r.clone()); - r - } + let result = if let Some(cached) = memo.get(&key) { + cached.clone() + } else { + let r = rt + .block_on(conn.resolve_drv_output(store_dir, &key.0, &key.1)) + .unwrap_or_else(|e| { + tracing::warn!("resolve_drv_output failed: {e}"); + None + }); + memo.insert(key, r.clone()); + r }; current = result?; } @@ -236,6 +223,8 @@ impl StepInfo { #[cfg(test)] mod tests { + #![allow(clippy::unwrap_used)] + use super::*; use db::models::BuildID; @@ -246,9 +235,9 @@ mod tests { lowest_share_used: f64, rdeps_len: u64, ) -> StepInfo { - let step = Step::new(nix_utils::parse_store_path( - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-test.drv", - )); + let step = Step::new( + StorePath::from_base_path("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-test.drv").unwrap(), + ); step.atomic_state .highest_global_priority diff --git a/subprojects/hydra-queue-runner/src/state/uploader.rs b/subprojects/hydra-queue-runner/src/state/uploader.rs index 8fb3a1a41..0b3b5dd13 100644 --- a/subprojects/hydra-queue-runner/src/state/uploader.rs +++ b/subprojects/hydra-queue-runner/src/state/uploader.rs @@ -1,6 +1,16 @@ use backon::ExponentialBuilder; use backon::Retryable as _; -use nix_utils::BaseStore as _; +use harmonia_store_path::StorePath; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum UploaderError { + #[error("uploader state I/O")] + Io(#[from] std::io::Error), + #[error("(de)serializing uploader state")] + Json(#[from] serde_json::Error), + #[error(transparent)] + Cache(#[from] binary_cache::CacheError), +} #[allow(clippy::unnecessary_wraps)] fn deserialize_with_new_v4<'de, D>(_: D) -> Result @@ -14,9 +24,9 @@ where struct Message { #[serde(skip_serializing, deserialize_with = "deserialize_with_new_v4")] id: uuid::Uuid, - store_paths: std::sync::Arc>, + store_paths: std::sync::Arc>, log_remote_path: std::sync::Arc, - log_local_path: std::sync::Arc, + log_local_path: std::sync::Arc, } #[derive(Debug)] @@ -46,7 +56,7 @@ impl Uploader { uploader } - async fn save_state(&self) -> anyhow::Result<()> { + async fn save_state(&self) -> Result<(), UploaderError> { let mut queue = self.queue.inspect(); queue.extend(self.current_tasks.read().iter().cloned()); let json = serde_json::to_string(&queue)?; @@ -55,7 +65,7 @@ impl Uploader { Ok(()) } - async fn load_state(&self) -> anyhow::Result<()> { + async fn load_state(&self) -> Result<(), UploaderError> { if !self.state_file_path.exists() { tracing::info!( "Uploader state file {} does not exist, starting with empty queue", @@ -77,9 +87,9 @@ impl Uploader { #[tracing::instrument(skip(self))] pub async fn schedule_upload( &self, - store_paths: Vec, + store_paths: Vec, log_remote_path: String, - log_local_path: String, + log_local_path: std::path::PathBuf, ) { tracing::info!("Scheduling new path upload: {:?}", store_paths); self.queue.send(Message { @@ -94,7 +104,7 @@ impl Uploader { #[tracing::instrument(skip(self, local_store, remote_stores))] async fn upload_msg( &self, - local_store: nix_utils::LocalStore, + local_store: harmonia_store_remote::ConnectionPool, remote_stores: Vec, msg: Message, ) { @@ -102,11 +112,13 @@ impl Uploader { let _ = span.enter(); tracing::info!("Start uploading {} paths", msg.store_paths.len()); - let paths_to_copy = match local_store - .query_requisites(&msg.store_paths.iter().collect::>(), true) - .await + let closure = match daemon_client_utils::query_closure( + &local_store, + msg.store_paths.as_ref(), + ) + .await { - Ok(paths) => paths, + Ok(c) => c, Err(e) => { tracing::error!("Failed to query requisites: {e}"); return; @@ -115,73 +127,92 @@ impl Uploader { tracing::info!( "{} paths results in {} paths_to_copy", msg.store_paths.len(), - paths_to_copy.len() + closure.len() ); for remote_store in remote_stores { - let bucket = &remote_store.cfg.client_config.bucket; - - // Upload log file with backon retry - let log_upload_result = (|| async { - let file = fs_err::tokio::File::open(msg.log_local_path.as_str()).await?; - let reader = Box::new(tokio::io::BufReader::new(file)); - - remote_store - .upsert_file_stream(&msg.log_remote_path, reader, "text/plain; charset=utf-8") - .await?; - - Ok::<(), anyhow::Error>(()) - }) - .retry( - ExponentialBuilder::default() - .with_max_delay(std::time::Duration::from_secs(30)) - .with_max_times(3), - ) - .await; - - if let Err(e) = log_upload_result { - tracing::error!("Failed to upload log file after retries: {e}"); - } - - let paths_to_copy = remote_store - .query_missing_paths(paths_to_copy.clone()) - .await; - tracing::info!( - "{} paths missing in remote store that we be copied", - paths_to_copy.len() - ); - - let copy_result = (|| async { - remote_store - .copy_paths(&local_store, paths_to_copy.clone(), false) - .await?; - - Ok::<(), anyhow::Error>(()) - }) - .retry( - ExponentialBuilder::default() - .with_max_delay(std::time::Duration::from_secs(60)) - .with_max_times(5), - ) - .await; - - if let Err(e) = copy_result { - tracing::error!("Failed to copy paths after retries: {e}"); - } else { - tracing::debug!( - "Successfully uploaded {} paths to bucket {bucket}", - msg.store_paths.len() + if let Err(e) = Self::upload_to_store(&remote_store, &local_store, &msg, &closure).await + { + // Non-fatal: per-store failure shouldn't block other + // stores. Outputs remain in the local store. + tracing::error!( + "Failed to upload to {}: {e:#}", + remote_store.cfg.client_config.bucket, ); } } - tracing::info!("Finished uploading {} paths", msg.store_paths.len()); + tracing::info!( + "Finished attempting to upload {} paths to remotes stores", + msg.store_paths.len() + ); + } + + /// Upload log + NARs to a single remote store. Returns `Err` if + /// anything goes wrong (after retries). + async fn upload_to_store( + remote_store: &binary_cache::S3BinaryCacheClient, + local_store: &harmonia_store_remote::ConnectionPool, + msg: &Message, + closure: &[harmonia_store_path_info::ValidPathInfo], + ) -> Result<(), UploaderError> { + // Upload log file + (|| async { + let file = fs_err::tokio::File::open(msg.log_local_path.as_path()).await?; + let reader = Box::new(tokio::io::BufReader::new(file)); + remote_store + .upsert_file_stream(&msg.log_remote_path, reader, "text/plain; charset=utf-8") + .await?; + Ok::<(), UploaderError>(()) + }) + .retry( + ExponentialBuilder::default() + .with_max_delay(std::time::Duration::from_secs(30)) + .with_max_times(3), + ) + .await?; + + // Copy NARs + let missing_paths: hashbrown::HashSet = remote_store + .query_missing_paths(closure.iter().map(|vpi| vpi.path.clone()).collect()) + .await + .into_iter() + .collect(); + let paths_to_copy: Vec<_> = closure + .iter() + .filter(|vpi| missing_paths.contains(&vpi.path)) + .cloned() + .collect(); + tracing::info!( + "{} paths missing in remote store that will be copied", + paths_to_copy.len() + ); + + (|| async { + remote_store + .copy_paths(local_store, paths_to_copy.clone(), false) + .await?; + Ok::<(), UploaderError>(()) + }) + .retry( + ExponentialBuilder::default() + .with_max_delay(std::time::Duration::from_mins(1)) + .with_max_times(5), + ) + .await?; + + tracing::debug!( + "Successfully uploaded {} paths to bucket {}", + msg.store_paths.len(), + remote_store.cfg.client_config.bucket, + ); + Ok(()) } #[tracing::instrument(skip(self, local_store, remote_stores))] pub async fn upload_once( &self, - local_store: nix_utils::LocalStore, + local_store: harmonia_store_remote::ConnectionPool, remote_stores: Vec, ) { let Some(msg) = self.queue.recv().await else { @@ -205,7 +236,7 @@ impl Uploader { #[tracing::instrument(skip(self, local_store, remote_stores))] pub async fn upload_many( &self, - local_store: nix_utils::LocalStore, + local_store: harmonia_store_remote::ConnectionPool, remote_stores: Vec, limit: usize, ) { @@ -237,7 +268,7 @@ impl Uploader { self.queue.len() } - pub fn paths_in_queue(&self) -> Vec { + pub fn paths_in_queue(&self) -> Vec { self.queue .inspect() .into_iter() diff --git a/subprojects/hydra-queue-runner/src/utils.rs b/subprojects/hydra-queue-runner/src/utils.rs index 5b7486493..8d11d00a7 100644 --- a/subprojects/hydra-queue-runner/src/utils.rs +++ b/subprojects/hydra-queue-runner/src/utils.rs @@ -1,20 +1,21 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, os::unix::ffi::OsStrExt as _}; use db::models::BuildID; -use nix_utils::{BaseStore as _, StorePath}; +use harmonia_store_derivation::derived_path::OutputName; +use harmonia_store_path::{StoreDir, StorePath}; -use crate::state::RemoteBuild; +use crate::state::{RemoteBuild, StateError}; -#[tracing::instrument(skip(db, store, res), err)] +#[tracing::instrument(skip(db, store_dir, res), err)] pub async fn finish_build_step( db: &db::Database, - store: &nix_utils::LocalStore, + store_dir: &StoreDir, build_id: BuildID, step_nr: i32, res: &RemoteBuild, machine: Option<&str>, - output_paths: Option<&BTreeMap>, -) -> anyhow::Result<()> { + output_paths: Option<&BTreeMap>, +) -> Result<(), StateError> { let mut conn = db.get().await?; let mut tx = conn.begin_transaction().await?; @@ -39,17 +40,21 @@ pub async fn finish_build_step( is_non_deterministic: res.get_is_non_deterministic(), }) .await?; - debug_assert!(!res.log_file.is_empty()); - debug_assert!(!res.log_file.contains('\t')); + debug_assert!(!res.log_file.as_os_str().is_empty()); + debug_assert!(!res.log_file.as_os_str().as_bytes().contains(&b'\t')); - tx.notify_step_finished(build_id, step_nr, &res.log_file) - .await?; + tx.notify_step_finished( + build_id, + step_nr, + res.log_file.to_str().ok_or(StateError::LogPathNotUtf8)?, + ) + .await?; if res.step_status == db::models::BuildStatus::Success && let Some(output_paths) = output_paths { for (name, path) in output_paths { - tx.update_build_step_output(store.store_dir(), build_id, step_nr, name.as_ref(), path) + tx.update_build_step_output(store_dir, build_id, step_nr, name.as_ref(), path) .await?; } } @@ -58,44 +63,56 @@ pub async fn finish_build_step( Ok(()) } -#[tracing::instrument(skip(db, store, remote_store), fields(%drv_path), err(level=tracing::Level::WARN))] +#[tracing::instrument(skip(db, pool, remote_store), fields(%drv_path), err(level=tracing::Level::WARN))] pub async fn substitute_output( db: db::Database, - store: nix_utils::LocalStore, - o: (nix_utils::OutputName, Option), + pool: harmonia_store_remote::ConnectionPool, + o: (OutputName, Option), build_id: BuildID, drv_path: &StorePath, remote_store: Option<&binary_cache::S3BinaryCacheClient>, -) -> anyhow::Result { +) -> Result { let (name, path) = o; let Some(path) = path else { return Ok(false); }; + let store_dir = pool.store_dir(); let starttime = i32::try_from(jiff::Timestamp::now().as_second())?; // TODO - if let Err(e) = store.ensure_path(&path).await { + if let Err(e) = daemon_client_utils::ensure_path(&pool, &path).await { tracing::debug!("Path not found, can't import={e}"); return Ok(false); } if let Some(remote_store) = remote_store { - let paths_to_copy = store - .query_requisites(&[&path], false) - .await - .unwrap_or_default(); - let paths_to_copy = remote_store.query_missing_paths(paths_to_copy).await; - if let Err(e) = remote_store.copy_paths(&store, paths_to_copy, false).await { + let _: Result<(), StateError> = async { + let closure = + daemon_client_utils::query_closure(&pool, std::slice::from_ref(&path)).await?; + let missing: hashbrown::HashSet = remote_store + .query_missing_paths(closure.iter().map(|vpi| vpi.path.clone()).collect()) + .await + .into_iter() + .collect(); + let paths_to_copy: Vec<_> = closure + .into_iter() + .filter(|vpi| missing.contains(&vpi.path)) + .collect(); + remote_store.copy_paths(&pool, paths_to_copy, false).await?; + Ok(()) + } + .await + .inspect_err(|e| { tracing::error!( "Failed to copy paths to remote store({}): {e}", remote_store.cfg.client_config.bucket ); - } + }); } let stoptime = i32::try_from(jiff::Timestamp::now().as_second())?; // TODO let mut db = db.get().await?; let mut tx = db.begin_transaction().await?; tx.create_substitution_step( - store.store_dir(), + store_dir, starttime, stoptime, build_id, @@ -108,20 +125,20 @@ pub async fn substitute_output( Ok(true) } -#[tracing::instrument(skip(db, store, ), fields(%drv_path), err(level=tracing::Level::WARN))] +#[tracing::instrument(skip(db, store_dir), fields(%drv_path), err(level=tracing::Level::WARN))] pub async fn make_local_step( db: &db::Database, - store: &nix_utils::LocalStore, + store_dir: &StoreDir, build_id: BuildID, drv_path: &StorePath, - missing: &BTreeMap>, -) -> anyhow::Result<()> { + missing: &BTreeMap>, +) -> Result<(), StateError> { let time = i32::try_from(jiff::Timestamp::now().as_second())?; let mut db = db.get().await?; let mut tx = db.begin_transaction().await?; tx.create_local_step( - store.store_dir(), + store_dir, time, time, build_id, diff --git a/subprojects/hydra-tests/content-addressed/basic.t b/subprojects/hydra-tests/content-addressed/basic.t index 3d7255ad0..3ad3d2108 100644 --- a/subprojects/hydra-tests/content-addressed/basic.t +++ b/subprojects/hydra-tests/content-addressed/basic.t @@ -24,7 +24,7 @@ my $project = $db->resultset('Projects')->create({name => "tests", displayname = my $jobset = createBaseJobset($db, "content-addressed", "content-addressed.nix", $ctx{jobsdir}); ok(evalSucceeds($ctx{context}, $jobset), "Evaluating jobs/content-addressed.nix should exit with return code 0"); -is(nrQueuedBuildsForJobset($jobset), 14, "Evaluating jobs/content-addressed.nix should result in 14 builds"); +is(nrQueuedBuildsForJobset($jobset), 10, "Evaluating jobs/content-addressed.nix should result in 10 builds"); my @builds = queuedBuildsForJobset($jobset); ok(runBuilds($ctx{context}, @builds), "Building all jobs from jobs/content-addressed.nix should exit with code 0"); @@ -58,40 +58,4 @@ for my $build (@builds) { # XXX: This test seems to not do what it seems to be doing. See documentation: https://metacpan.org/pod/Test2::V0#isnt($got,-$do_not_want,-$name) isnt(<$ctx{deststoredir}/realisations/*>, "", "The destination store should have the realisations of the built derivations registered"); -# Early cutoff: earlyCutoffUpstream1 and earlyCutoffUpstream2 have -# different derivations but produce the same content-addressed output. -# After building earlyCutoffDownstream1, earlyCutoffDownstream2 should -# be cached because its resolved input is identical. -my $upstream1 = $db->resultset('Builds')->find({ - jobset_id => $jobset->id, - job => "earlyCutoffUpstream1", -}); -my $upstream2 = $db->resultset('Builds')->find({ - jobset_id => $jobset->id, - job => "earlyCutoffUpstream2", -}); - -my $upstream1_out = $upstream1->buildoutputs->find({ name => "out" }); -my $upstream2_out = $upstream2->buildoutputs->find({ name => "out" }); -is($upstream1_out->path, $upstream2_out->path, - "Both upstream builds should resolve to the same content-addressed output path"); - -my $downstream1 = $db->resultset('Builds')->find({ - jobset_id => $jobset->id, - job => "earlyCutoffDownstream1", -}); -my $downstream2 = $db->resultset('Builds')->find({ - jobset_id => $jobset->id, - job => "earlyCutoffDownstream2", -}); - -my $downstream1_out = $downstream1->buildoutputs->find({ name => "out" }); -my $downstream2_out = $downstream2->buildoutputs->find({ name => "out" }); -is($downstream1_out->path, $downstream2_out->path, - "Both downstream builds should create the same content-addressed output path"); - -ok($downstream1->iscachedbuild || $downstream2->iscachedbuild, - "One downstream build should be cached"); - done_testing; - diff --git a/subprojects/hydra-tests/content-addressed/early-cutoff.t b/subprojects/hydra-tests/content-addressed/early-cutoff.t new file mode 100644 index 000000000..a5d4c4835 --- /dev/null +++ b/subprojects/hydra-tests/content-addressed/early-cutoff.t @@ -0,0 +1,71 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; + +my %ctx = test_init( + nix_config => qq| + experimental-features = ca-derivations + |, +); + +use Test2::V0; + +my $db = $ctx{context}->db(); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset($db, "content-addressed-early-cutoff", "content-addressed-early-cutoff.nix", $ctx{jobsdir}); + +ok(evalSucceeds($ctx{context}, $jobset), "Evaluating early cutoff jobs should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 4, "Should queue 4 early cutoff builds"); + +my @builds = queuedBuildsForJobset($jobset); +ok(runBuilds($ctx{context}, @builds), "Building all early cutoff jobs should exit with code 0"); + +for my $build (@builds) { + my $newbuild = $db->resultset('Builds')->find($build->id); + is($newbuild->finished, 1, "Build '".$build->job."' should be finished."); + is($newbuild->buildstatus, 0, "Build '".$build->job."' should have buildstatus 0."); +} + +# Early cutoff: earlyCutoffUpstream1 and earlyCutoffUpstream2 have +# different derivations but produce the same content-addressed output. +# After building earlyCutoffDownstream1, earlyCutoffDownstream2 should +# be cached because its resolved input is identical. +my $upstream1 = $db->resultset('Builds')->find({ + jobset_id => $jobset->id, + job => "earlyCutoffUpstream1", +}); +my $upstream2 = $db->resultset('Builds')->find({ + jobset_id => $jobset->id, + job => "earlyCutoffUpstream2", +}); + +my $upstream1_out = $upstream1->buildoutputs->find({ name => "out" }); +my $upstream2_out = $upstream2->buildoutputs->find({ name => "out" }); +is($upstream1_out->path, $upstream2_out->path, + "Both upstream builds should resolve to the same content-addressed output path"); + +my $downstream1 = $db->resultset('Builds')->find({ + jobset_id => $jobset->id, + job => "earlyCutoffDownstream1", +}); +my $downstream2 = $db->resultset('Builds')->find({ + jobset_id => $jobset->id, + job => "earlyCutoffDownstream2", +}); + +my $downstream1_out = $downstream1->buildoutputs->find({ name => "out" }); +my $downstream2_out = $downstream2->buildoutputs->find({ name => "out" }); +is($downstream1_out->path, $downstream2_out->path, + "Both downstream builds should create the same content-addressed output path"); + +# Skip this for now because it is currently possible for neither to be marked +# cached. Further investigation is needed. Note that it is not clear that the +# derivation is being built twice in the neither-marked-cached case either! +# +#ok($downstream1->iscachedbuild || $downstream2->iscachedbuild, +# "One downstream build should be cached"); + +done_testing; diff --git a/subprojects/hydra-tests/jobs/config.nix.in b/subprojects/hydra-tests/jobs/config.nix.in index 80889fb18..314c72b59 100644 --- a/subprojects/hydra-tests/jobs/config.nix.in +++ b/subprojects/hydra-tests/jobs/config.nix.in @@ -2,10 +2,11 @@ rec { path = "@testPath@"; nixBinDir = "@nixBinDir@"; bash = "@bash@"; + system = "@system@"; mkDerivation = args: derivation ({ - system = builtins.currentSystem; + inherit system; PATH = path; } // args); mkContentAddressedDerivation = args: mkDerivation ({ diff --git a/subprojects/hydra-tests/jobs/content-addressed-early-cutoff.nix b/subprojects/hydra-tests/jobs/content-addressed-early-cutoff.nix new file mode 100644 index 000000000..1ab775382 --- /dev/null +++ b/subprojects/hydra-tests/jobs/content-addressed-early-cutoff.nix @@ -0,0 +1,33 @@ +let + cfg = import ./config.nix; +in +rec { + # Early-cutoff test: two upstream derivations that differ in a dummy + # attribute but produce the same output (an empty directory). Because + # they are content-addressed, they resolve to the same output path. + # Both downstreams depend on a different upstream, but since the + # resolved input is identical, the second downstream should be cached. + earlyCutoffUpstream1 = cfg.mkContentAddressedDerivation { + name = "early-cutoff-upstream"; + builder = ./empty-dir-builder.sh; + dummy = "1"; + }; + + earlyCutoffUpstream2 = cfg.mkContentAddressedDerivation { + name = "early-cutoff-upstream"; + builder = ./empty-dir-builder.sh; + dummy = "2"; + }; + + earlyCutoffDownstream1 = cfg.mkContentAddressedDerivation { + name = "early-cutoff-downstream"; + builder = ./dir-with-file-builder.sh; + FOO = earlyCutoffUpstream1; + }; + + earlyCutoffDownstream2 = cfg.mkContentAddressedDerivation { + name = "early-cutoff-downstream"; + builder = ./dir-with-file-builder.sh; + FOO = earlyCutoffUpstream2; + }; +} diff --git a/subprojects/hydra-tests/jobs/content-addressed.nix b/subprojects/hydra-tests/jobs/content-addressed.nix index 9960a4cb3..4e208311a 100644 --- a/subprojects/hydra-tests/jobs/content-addressed.nix +++ b/subprojects/hydra-tests/jobs/content-addressed.nix @@ -64,33 +64,4 @@ rec { "lib" ]; }; - - # Early-cutoff test: two upstream derivations that differ in a dummy - # attribute but produce the same output (an empty directory). Because - # they are content-addressed, they resolve to the same output path. - # Both downstreams depend on a different upstream, but since the - # resolved input is identical, the second downstream should be cached. - earlyCutoffUpstream1 = cfg.mkContentAddressedDerivation { - name = "early-cutoff-upstream"; - builder = ./empty-dir-builder.sh; - dummy = "1"; - }; - - earlyCutoffUpstream2 = cfg.mkContentAddressedDerivation { - name = "early-cutoff-upstream"; - builder = ./empty-dir-builder.sh; - dummy = "2"; - }; - - earlyCutoffDownstream1 = cfg.mkContentAddressedDerivation { - name = "early-cutoff-downstream"; - builder = ./dir-with-file-builder.sh; - FOO = earlyCutoffUpstream1; - }; - - earlyCutoffDownstream2 = cfg.mkContentAddressedDerivation { - name = "early-cutoff-downstream"; - builder = ./dir-with-file-builder.sh; - FOO = earlyCutoffUpstream2; - }; } diff --git a/subprojects/hydra-tests/jobs/logging-failure.sh b/subprojects/hydra-tests/jobs/logging-failure.sh new file mode 100755 index 000000000..a69995ede --- /dev/null +++ b/subprojects/hydra-tests/jobs/logging-failure.sh @@ -0,0 +1,5 @@ +#! /bin/sh + +echo "should appear in failure" + +exit 1 diff --git a/subprojects/hydra-tests/jobs/logging-success.sh b/subprojects/hydra-tests/jobs/logging-success.sh new file mode 100755 index 000000000..78fa0ddb7 --- /dev/null +++ b/subprojects/hydra-tests/jobs/logging-success.sh @@ -0,0 +1,5 @@ +#! /bin/sh + +echo "should appear in success" +mkdir $out +echo "is output" > $out/file diff --git a/subprojects/hydra-tests/jobs/logging.nix b/subprojects/hydra-tests/jobs/logging.nix new file mode 100644 index 000000000..4e3118987 --- /dev/null +++ b/subprojects/hydra-tests/jobs/logging.nix @@ -0,0 +1,12 @@ +with import ./config.nix; +{ + success = mkDerivation { + name = "logging-success"; + builder = ./logging-success.sh; + }; + + failure = mkDerivation { + name = "logging-failure"; + builder = ./logging-failure.sh; + }; +} diff --git a/subprojects/hydra-tests/lib/HydraTestContext.pm b/subprojects/hydra-tests/lib/HydraTestContext.pm index 58d604b5f..25cae4095 100644 --- a/subprojects/hydra-tests/lib/HydraTestContext.pm +++ b/subprojects/hydra-tests/lib/HydraTestContext.pm @@ -104,10 +104,12 @@ sub new { my $coreutils_path = dirname(which 'install'); my $nix_bin_dir = dirname(which 'nix'); my $bash_path = which 'bash'; + chomp(my $system = `nix --extra-experimental-features nix-command config show system`); replace_variable_in_file($jobsdir . "/config.nix", '@testPath@' => $coreutils_path, '@nixBinDir@' => $nix_bin_dir, - '@bash@' => $bash_path); + '@bash@' => $bash_path, + '@system@' => $system); replace_variable_in_file($jobsdir . "/declarative/project.json", '@jobsPath@' => $jobsdir); diff --git a/subprojects/hydra-tests/lib/NixDaemon.pm b/subprojects/hydra-tests/lib/NixDaemon.pm new file mode 100644 index 000000000..011ecca93 --- /dev/null +++ b/subprojects/hydra-tests/lib/NixDaemon.pm @@ -0,0 +1,32 @@ +use warnings; +use strict; + +package NixDaemon; +use File::Path qw(make_path); +our @ISA = qw(Exporter); +our @EXPORT_OK = qw(start_nix_daemon); + +# Start a nix daemon for the given store config and register it in a +# ProcessGroup so its logs are pumped alongside the other processes. +sub start_nix_daemon { + my ($store, $pg, $label) = @_; + make_path($store->{nix_state_dir}); + + my $harness = $pg->spawn($label, ["nix-daemon"], env => { + NIX_REMOTE => $store->{nix_store_uri}, + NIX_STORE_DIR => $store->{nix_store_dir}, + NIX_STATE_DIR => $store->{nix_state_dir}, + NIX_CONF_DIR => $store->{nix_conf_dir}, + NIX_DAEMON_SOCKET_PATH => $store->{nix_daemon_socket_path}, + NIX_CONFIG => "trusted-users = *", + }); + my $socket = $store->{nix_daemon_socket_path}; + for (1..50) { + last if -S $socket; + select(undef, undef, undef, 0.1); + } + -S $socket or die "nix-daemon did not start: $socket\n"; + return $harness; +} + +1; diff --git a/subprojects/hydra-tests/lib/ProcessGroup.pm b/subprojects/hydra-tests/lib/ProcessGroup.pm new file mode 100644 index 000000000..dfa44b536 --- /dev/null +++ b/subprojects/hydra-tests/lib/ProcessGroup.pm @@ -0,0 +1,123 @@ +use warnings; +use strict; + +package ProcessGroup; + +# A lightweight wrapper around a set of IPC::Run harnesses with +# labelled log flushing, pump, and ordered teardown. +# +# Usage: +# my $pg = ProcessGroup->new; +# $pg->spawn("queue-runner", ["hydra-queue-runner", ...], env => { ... }); +# $pg->pump_logs; +# $pg->stop; # or let DESTROY handle it + +use IPC::Run; + +our @ISA = qw(Exporter); +our @EXPORT_OK = qw(); + +sub new { + my ($class, %opts) = @_; + bless { + procs => {}, + order => [], + env => $opts{env} // {}, + }, $class; +} + +# Spawn a labelled child process and register it. +# +# Options: +# env => { KEY => VAL, ... } extra env vars merged with the group default +# init => sub { ... } passed to IPC::Run::start as init callback +sub spawn { + my ($self, $key, $cmd, %opts) = @_; + my %env = (%{$self->{env}}, %{$opts{env} // {}}); + my ($in, $out, $err) = ("", "", ""); + my $harness; + { + local @ENV{keys %env} = values %env; + local $ENV{NO_COLOR} = "1"; + my @extra; + push @extra, (init => $opts{init}) if $opts{init}; + $harness = IPC::Run::start($cmd, \$in, \$out, \$err, @extra); + } + $self->{procs}{$key} = { + label => $key, + harness => $harness, + out => \$out, + err => \$err, + }; + push @{$self->{order}}, $key; + return $harness; +} + +# Pump all harnesses and flush any complete log lines to stderr. +sub pump_logs { + my ($self) = @_; + for my $key (@{$self->{order}}) { + my $p = $self->{procs}{$key} or next; + eval { $p->{harness}->pump_nb }; + _flush_proc($p); + } +} + +# Return the harness for a given key, or undef. +sub harness { + my ($self, $key) = @_; + my $p = $self->{procs}{$key} or return undef; + return $p->{harness}; +} + +# Return the stdout buffer ref for a given key. +sub stdout_ref { + my ($self, $key) = @_; + return $self->{procs}{$key}{out}; +} + +# Return the stderr buffer ref for a given key. +sub stderr_ref { + my ($self, $key) = @_; + return $self->{procs}{$key}{err}; +} + +# Kill all processes in reverse spawn order and flush final output. +sub stop { + my ($self) = @_; + return if $self->{stopped}; + $self->{stopped} = 1; + for my $key (reverse @{$self->{order}}) { + my $p = $self->{procs}{$key} or next; + eval { $p->{harness}->kill_kill }; + _flush_proc($p, 1); + } +} + +sub DESTROY { + my ($self) = @_; + $self->stop; +} + +# --- Internal helpers --- + +sub _flush_stream { + my ($label, $stream, $buf_ref, $final) = @_; + return if $$buf_ref eq ""; + utf8::decode($$buf_ref) or warn "Invalid unicode in $label $stream."; + while ($$buf_ref =~ s/^([^\n]*)\n//) { + print STDERR "[$label $stream] $1\n"; + } + if ($final && $$buf_ref ne "") { + print STDERR "[$label $stream] $$buf_ref\n"; + $$buf_ref = ""; + } +} + +sub _flush_proc { + my ($p, $final) = @_; + _flush_stream($p->{label}, "stdout", $p->{out}, $final); + _flush_stream($p->{label}, "stderr", $p->{err}, $final); +} + +1; diff --git a/subprojects/hydra-tests/lib/QueueRunnerBuildOne.pm b/subprojects/hydra-tests/lib/QueueRunnerBuildOne.pm index 65db8e92e..767947749 100644 --- a/subprojects/hydra-tests/lib/QueueRunnerBuildOne.pm +++ b/subprojects/hydra-tests/lib/QueueRunnerBuildOne.pm @@ -2,10 +2,11 @@ use warnings; use strict; package QueueRunnerBuildOne; -use IPC::Run; use JSON::PP; use LWP::UserAgent; use HTTP::Request; +use NixDaemon qw(start_nix_daemon); +use ProcessGroup; use QueueRunnerContext; our @ISA = qw(Exporter); our @EXPORT = qw( @@ -13,77 +14,42 @@ our @EXPORT = qw( runBuilds ); -sub _flush_stream { - my ($label, $stream, $buf_ref, $final) = @_; - return if $$buf_ref eq ""; - utf8::decode($$buf_ref) or warn "Invalid unicode in $label $stream."; - # Print each complete line with a label prefix. Leave any trailing - # partial line in the buffer so it can be flushed together with the - # rest of its line on a subsequent call. - while ($$buf_ref =~ s/^([^\n]*)\n//) { - print STDERR "[$label $stream] $1\n"; - } - if ($final && $$buf_ref ne "") { - print STDERR "[$label $stream] $$buf_ref\n"; - $$buf_ref = ""; - } -} - -sub _flush_harness { - my ($label, $out_ref, $err_ref, $final) = @_; - _flush_stream($label, "stdout", $out_ref, $final); - _flush_stream($label, "stderr", $err_ref, $final); -} - sub runBuilds { my ($ctx, @builds) = @_; ref $ctx eq 'HydraTestContext' or die "runBuilds requires a HydraTestContext as first argument\n"; my @build_ids = map { $_->id } @builds; - my ($qr_harness, $base_url, $grpc_addr, $qr_out_ref, $qr_err_ref, $qr_daemon) = start_queue_runner($ctx, - rust_log => "queue_runner=debug,info", - ); + my ($pg, $base_url, $grpc_addr) = start_queue_runner($ctx); - # Start a nix daemon for the builder. - my $bl_daemon = QueueRunnerContext::start_nix_daemon($ctx->{builder}); + start_nix_daemon($ctx->{builder}, $pg, "builder daemon"); - my ($bl_in, $bl_out, $bl_err) = ("", "", ""); - my $ua = LWP::UserAgent->new(timeout => 2); - my $bl_harness; + $pg->spawn("builder", + ["hydra-builder", "--gateway-endpoint", "http://$grpc_addr"], + env => { + NIX_REMOTE => $ctx->{builder}{nix_daemon_uri}, + NIX_CONF_DIR => $ctx->{builder}{nix_conf_dir}, + NIX_STATE_DIR => $ctx->{builder}{nix_state_dir}, + NIX_STORE_DIR => $ctx->{builder}{nix_store_dir}, + RUST_LOG => "hydra_builder=debug,info", + }, + ); + my $ua = LWP::UserAgent->new(timeout => 2); my $timeout = 60 * scalar(@build_ids); + my $ok = eval { local $SIG{ALRM} = sub { die "timeout\n" }; alarm $timeout; - # Wait for the REST server to become available. + wait_for_url($ua, "$base_url/status") or die "Timed out waiting for queue-runner REST server\n"; - # Start the builder, connecting to the nix daemon via unix://. - { - local $ENV{NIX_REMOTE} = $ctx->{builder}{nix_daemon_uri}; - local $ENV{NIX_CONF_DIR} = $ctx->{builder}{nix_conf_dir}; - local $ENV{NIX_STATE_DIR} = $ctx->{builder}{nix_state_dir}; - # TODO: hydra-builder reads NIX_STORE_DIR to report its - # store dir to the queue runner; should use the store URI instead. - local $ENV{NIX_STORE_DIR} = $ctx->{builder}{nix_store_dir}; - local $ENV{RUST_LOG} = "hydra_builder=debug,info"; - local $ENV{NO_COLOR} = "1"; - $bl_harness = IPC::Run::start( - ["hydra-builder", - "--gateway-endpoint", "http://$grpc_addr", - ], - \$bl_in, \$bl_out, \$bl_err, - ); - } - - # Wait for the builder to register as a machine. wait_for_url($ua, "$base_url/status/machines", sub { shift->decoded_content =~ /"hostname"/; }) or die "Timed out waiting for builder to register\n"; - # Submit all builds. Returns 200 even if a build is already finished. for my $bid (@build_ids) { + $pg->pump_logs; my $req = HTTP::Request->new(POST => "$base_url/build_one"); $req->header('Content-Type' => 'application/json'); $req->content(encode_json({ buildId => $bid + 0 })); @@ -92,33 +58,7 @@ sub runBuilds { unless $resp->is_success; } - # Poll until every build is no longer active. - while (1) { - # If the builder crashed, fail fast instead of waiting for the - # queue-runner to time out the orphaned builds. - $qr_harness->pump_nb; - $bl_harness->pump_nb; - # Flush accumulated output so logs are visible while waiting. - _flush_harness("Queue runner", $qr_out_ref, $qr_err_ref); - _flush_harness("Builder", \$bl_out, \$bl_err); - if (!$bl_harness->pumpable) { - $bl_harness->finish; - my $rc = $bl_harness->result; - print STDERR "builder exited unexpectedly (exit code $rc)\n"; - die "builder exited unexpectedly\n"; - } - - my $all_done = 1; - for my $bid (@build_ids) { - my $resp = $ua->get("$base_url/status/build/$bid/active"); - if ($resp->decoded_content =~ /true/) { - $all_done = 0; - last; - } - } - last if $all_done; - sleep 2; - } + wait_for_builds($ua, $base_url, $pg, @build_ids); alarm 0; 1; @@ -126,16 +66,7 @@ sub runBuilds { my $err = $@; alarm 0; - # Always clean up child processes, then flush any trailing stderr - # that arrived after the last poll iteration. - if ($bl_harness) { - $bl_harness->kill_kill; - _flush_harness("Builder", \$bl_out, \$bl_err, 1); - } - $bl_daemon->kill_kill(grace => 2); - $qr_harness->kill_kill; - _flush_harness("Queue runner", $qr_out_ref, $qr_err_ref, 1); - $qr_daemon->kill_kill(grace => 2); + $pg->stop; if (!$ok) { print STDERR "runBuilds failed: $err" if $err; diff --git a/subprojects/hydra-tests/lib/QueueRunnerContext.pm b/subprojects/hydra-tests/lib/QueueRunnerContext.pm index 78c04fb54..90c0780bd 100644 --- a/subprojects/hydra-tests/lib/QueueRunnerContext.pm +++ b/subprojects/hydra-tests/lib/QueueRunnerContext.pm @@ -2,15 +2,17 @@ use warnings; use strict; package QueueRunnerContext; -use File::Path qw(make_path); use IO::Socket::IP; use IPC::Run; use LWP::UserAgent; use POSIX qw(dup2); use Hydra::Config; +use NixDaemon qw(start_nix_daemon); +use ProcessGroup; our @ISA = qw(Exporter); our @EXPORT = qw( start_queue_runner + wait_for_builds wait_for_url ); @@ -26,46 +28,21 @@ sub wait_for_url { return 0; } -# Start a nix daemon for the given store config. -# Returns the daemon harness. Caller must kill_kill it when done. -sub start_nix_daemon { - my ($store) = @_; - make_path($store->{nix_state_dir}); - - my ($in, $out, $err) = ("", "", ""); - my $harness; - { - local $ENV{NIX_REMOTE} = $store->{nix_store_uri}; - local $ENV{NIX_STORE_DIR} = $store->{nix_store_dir}; - local $ENV{NIX_STATE_DIR} = $store->{nix_state_dir}; - local $ENV{NIX_CONF_DIR} = $store->{nix_conf_dir}; - local $ENV{NIX_DAEMON_SOCKET_PATH} = $store->{nix_daemon_socket_path}; - local $ENV{NIX_CONFIG} = "trusted-users = *"; - $harness = IPC::Run::start( - ["nix-daemon"], - \$in, \$out, \$err, - ); - } - my $socket = $store->{nix_daemon_socket_path}; - for (1..50) { - last if -S $socket; - select(undef, undef, undef, 0.1); - } - -S $socket or die "nix-daemon did not start: $socket\n"; - return $harness; -} # Start a queue runner process using systemd socket activation. # We bind TCP sockets ourselves (port 0 for OS-assigned ports), then # pass them to the queue runner via LISTEN_FDS/LISTEN_FDNAMES. -# Returns ($harness, $rest_url, $grpc_addr, \$stdout_buf, \$stderr_buf, $daemon_harness). -# Caller is responsible for calling $harness->kill_kill when done. +# Returns ($process_group, $rest_url, $grpc_addr). +# The ProcessGroup has the nix daemon and queue-runner registered. +# Caller is responsible for calling $pg->stop when done. sub start_queue_runner { my ($ctx, %opts) = @_; ref $ctx eq 'HydraTestContext' or die "start_queue_runner requires a HydraTestContext\n"; + my $pg = ProcessGroup->new; + # Start a nix daemon for the queue runner to use. - my $daemon_harness = start_nix_daemon($ctx->{central}); + start_nix_daemon($ctx->{central}, $pg, "queue-runner daemon"); my $config_dir = $ENV{T2_HARNESS_TEMP_DIR} // $ctx->{central}{hydra_data}; @@ -124,7 +101,7 @@ sub start_queue_runner { { local @ENV{keys %{$ctx->{central_env}}} = values %{$ctx->{central_env}}; local $ENV{NIX_REMOTE} = $ctx->{central}{nix_daemon_uri}; - local $ENV{RUST_LOG} = $opts{rust_log} // "error"; + local $ENV{RUST_LOG} = "hydra_queue_runner=debug,info"; local $ENV{NO_COLOR} = "1"; local $ENV{LISTEN_FDS} = "2"; local $ENV{LISTEN_FDNAMES} = "rest:grpc"; @@ -156,7 +133,45 @@ sub start_queue_runner { my $rest_url = "http://[::1]:$rest_port"; my $grpc_addr = "[::1]:$grpc_port"; - return ($qr_harness, $rest_url, $grpc_addr, \$qr_out, \$qr_err, $daemon_harness); + $pg->{procs}{"queue-runner"} = { + label => "queue-runner", harness => $qr_harness, out => \$qr_out, err => \$qr_err, + }; + push @{$pg->{order}}, "queue-runner"; + + return ($pg, $rest_url, $grpc_addr); +} + +# Poll the queue runner REST API until all given build IDs are no longer +# active. Calls $pg->pump_logs each iteration so test output stays visible. +# +# Args: ($ua, $base_url, $process_group, @build_ids) +# Dies on timeout or if the builder process exits unexpectedly. +sub wait_for_builds { + my ($ua, $base_url, $pg, @build_ids) = @_; + my $timeout = 60 * scalar(@build_ids); + $timeout = 60 if $timeout < 60; + my $deadline = time() + $timeout; + while (time() < $deadline) { + $pg->pump_logs; + my $bl = $pg->harness("builder"); + if ($bl && !$bl->pumpable) { + $bl->finish; + my $rc = $bl->result; + die "builder exited unexpectedly (exit code $rc)\n"; + } + + my $all_done = 1; + for my $bid (@build_ids) { + my $resp = $ua->get("$base_url/status/build/$bid/active"); + if ($resp->decoded_content =~ /true/) { + $all_done = 0; + last; + } + } + return 1 if $all_done; + sleep 2; + } + die "timed out waiting for builds to finish\n"; } 1; diff --git a/subprojects/hydra-tests/queue-runner/logging.t b/subprojects/hydra-tests/queue-runner/logging.t new file mode 100644 index 000000000..b3f5279b9 --- /dev/null +++ b/subprojects/hydra-tests/queue-runner/logging.t @@ -0,0 +1,40 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; +use File::Slurper qw(read_text); +require Hydra::Helper::Nix; + +my $ctx = test_context(); +setup_catalyst_test($ctx); + +my $builds = $ctx->makeAndEvaluateJobset( + expression => "logging.nix", + build => 1 +); + +subtest "success" => sub { + my $build = $builds->{"success"}; + is($build->finished, 1, "Build should be finished."); + is($build->buildstatus, 0, "Build should have succeeded."); + + # getDrvLogPath wants HYDRA_DATA + local $ENV{HYDRA_DATA} = $ctx->{central}{hydra_data} . "/data"; + my $logPath = Hydra::Helper::Nix::getDrvLogPath($build->drvpath) or die "Log file did not exist"; + my $logContent = read_text($logPath) or die "Could not read log file"; + ok(index($logContent, "should appear in success") != -1, "Log should contain correct content"); +}; + +subtest "failure" => sub { + my $build = $builds->{"failure"}; + is($build->finished, 1, "Build should be finished."); + is($build->buildstatus, 1, "Build should have failed."); + + # getDrvLogPath wants HYDRA_DATA + local $ENV{HYDRA_DATA} = $ctx->{central}{hydra_data} . "/data"; + my $logPath = Hydra::Helper::Nix::getDrvLogPath($build->drvpath) or die "Log file did not exist"; + my $logContent = read_text($logPath) or die "Could not read log file"; + ok(index($logContent, "should appear in failure") != -1, "Log should contain correct content"); +}; + +done_testing; diff --git a/subprojects/hydra-tests/scripts/hydra-send-stats.t b/subprojects/hydra-tests/scripts/hydra-send-stats.t index e42d8689d..ee813c0b4 100644 --- a/subprojects/hydra-tests/scripts/hydra-send-stats.t +++ b/subprojects/hydra-tests/scripts/hydra-send-stats.t @@ -22,7 +22,7 @@ subtest "without queue_runner_endpoint" => sub { }; subtest "with queue_runner_endpoint" => sub { - my ($qr_harness, $base_url, undef, undef, undef, $daemon_harness) = start_queue_runner($ctx{context}); + my ($pg, $base_url) = start_queue_runner($ctx{context}); my $ok = eval { local $SIG{ALRM} = sub { die "timeout\n" }; @@ -54,8 +54,7 @@ subtest "with queue_runner_endpoint" => sub { my $err = $@; alarm 0; - $qr_harness->kill_kill; - $daemon_harness->kill_kill; + $pg->stop; die "with queue_runner_endpoint failed: $err" if !$ok && $err; }; diff --git a/subprojects/hydra-tests/test.pl b/subprojects/hydra-tests/test.pl index 122846373..ac4fa7018 100644 --- a/subprojects/hydra-tests/test.pl +++ b/subprojects/hydra-tests/test.pl @@ -21,7 +21,20 @@ BEGIN print STDERR "test.pl: Defaulting \$YATH_JOB_COUNT to \$NIX_BUILD_CORES (${\$ENV{'NIX_BUILD_CORES'}})\n"; } -system($^X, find_yath(), '-D', 'test', '--qvf', '--event-timeout', 240, '--default-search' => './', @ARGV); +system( + $^X, find_yath(), '-D', 'test', + '--qvf', + '--event-timeout', 240, + # Tests can be flaky in CI due to timing, resource contention, etc. + # Individual tests can still override with `# HARNESS-RETRY N`. + # + # TODO Clean this up. E.g. make hydra more robust, make CI builders bigger + # if needed, separate resource issues from real bugs and only retry on the + # former. + '--retry', 2, + '--default-search' => './', + @ARGV, +); my $exit = $?; # This makes sure it works with prove. diff --git a/subprojects/hydra/lib/Hydra/Controller/API.pm b/subprojects/hydra/lib/Hydra/Controller/API.pm index e5075fa08..7d26776f8 100644 --- a/subprojects/hydra/lib/Hydra/Controller/API.pm +++ b/subprojects/hydra/lib/Hydra/Controller/API.pm @@ -57,10 +57,10 @@ sub latestbuilds : Chained('api') PathPart('latestbuilds') Args(0) { my $system = $c->request->params->{system}; my $filter = {finished => 1}; - $filter->{"jobset.project"} = $project if ! $project eq ""; - $filter->{"jobset.name"} = $jobset if ! $jobset eq ""; - $filter->{job} = $job if !$job eq ""; - $filter->{system} = $system if !$system eq ""; + $filter->{"jobset.project"} = $project if $project ne ""; + $filter->{"jobset.name"} = $jobset if $jobset ne ""; + $filter->{job} = $job if $job ne ""; + $filter->{system} = $system if $system ne ""; my @latest = $c->model('DB::Builds')->search( $filter, @@ -163,10 +163,10 @@ sub nrbuilds : Chained('api') PathPart('nrbuilds') Args(0) { my $system = $c->request->params->{system}; my $filter = {finished => 1}; - $filter->{"jobset.project"} = $project if ! $project eq ""; - $filter->{"jobset.name"} = $jobset if ! $jobset eq ""; - $filter->{job} = $job if !$job eq ""; - $filter->{system} = $system if !$system eq ""; + $filter->{"jobset.project"} = $project if $project ne ""; + $filter->{"jobset.name"} = $jobset if $jobset ne ""; + $filter->{job} = $job if $job ne ""; + $filter->{system} = $system if $system ne ""; $base = 60*60 if($period eq "hour"); $base = 24*60*60 if($period eq "day"); diff --git a/subprojects/hydra/lib/Hydra/Plugin/GitlabStatus.pm b/subprojects/hydra/lib/Hydra/Plugin/GitlabStatus.pm index 9d6718d26..65c8c8922 100644 --- a/subprojects/hydra/lib/Hydra/Plugin/GitlabStatus.pm +++ b/subprojects/hydra/lib/Hydra/Plugin/GitlabStatus.pm @@ -38,7 +38,7 @@ sub toGitlabState { } sub common { - my ($self, $topbuild, $dependents, $status) = @_; + my ($self, $topbuild, $dependents, $status, $cachedEval) = @_; my $baseurl = $self->{config}->{'base_uri'} || "http://localhost:3000"; # Find matching configs @@ -59,6 +59,10 @@ sub common { name => "Hydra " . $build->get_column('job'), }); while (my $eval = $evals->next) { + if (defined($cachedEval) && $cachedEval->id != $eval->id) { + next; + } + my $gitlabstatusInput = $eval->jobsetevalinputs->find({ name => "gitlab_status_repo" }); next unless defined $gitlabstatusInput && defined $gitlabstatusInput->value; my $i = $eval->jobsetevalinputs->find({ name => $gitlabstatusInput->value, altnr => 0 }); @@ -68,7 +72,6 @@ sub common { my $rev = $i->revision; my $domain = URI->new($i->uri)->host; my $url = "https://$domain/api/v4/projects/$projectId/statuses/$rev"; - print STDERR "GitlabStatus POSTing $state to $url\n"; my $req = HTTP::Request->new('POST', $url); $req->header('Content-Type' => 'application/json'); $req->header('Private-Token' => $accessToken); @@ -91,4 +94,14 @@ sub buildFinished { common(@_, 2); } +sub cachedBuildQueued { + my ($self, $evaluation, $build) = @_; + common($self, $build, [], 0, $evaluation); +} + +sub cachedBuildFinished { + my ($self, $evaluation, $build) = @_; + common($self, $build, [], 2, $evaluation); +} + 1; diff --git a/subprojects/hydra/script/hydra-eval-jobset b/subprojects/hydra/script/hydra-eval-jobset index eeaab5fda..811b0a7a2 100755 --- a/subprojects/hydra/script/hydra-eval-jobset +++ b/subprojects/hydra/script/hydra-eval-jobset @@ -358,21 +358,14 @@ sub evalJobs { my @cmd; if (defined $flakeRef) { - my $nix_expr = - "let " . - "flake = builtins.getFlake (toString \"$flakeRef\"); " . - "in " . - "flake.hydraJobs " . - "or flake.checks " . - "or (throw \"flake '$flakeRef' does not provide any Hydra jobs or checks\")"; - @cmd = ("nix-eval-jobs", # Disable the eval cache to prevent SQLite database contention. # Since Hydra typically evaluates each revision only once, # parallel workers would compete for database locks without # providing any benefit from caching. "--option", "eval-cache", "false", - "--expr", $nix_expr); + "--flake", $flakeRef, + "--select", "flake: flake.outputs.hydraJobs or flake.outputs.checks or (throw \"flake does not provide any Hydra jobs or checks\")"); } else { my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] or die "cannot find the input containing the job expression\n"; diff --git a/subprojects/hydra/sql/README.md b/subprojects/hydra/sql/README.md index ddc1f51ff..d5172e8b4 100644 --- a/subprojects/hydra/sql/README.md +++ b/subprojects/hydra/sql/README.md @@ -38,5 +38,4 @@ ./schemas/verify.sh ``` - (Note, this requires a full git history, not just the latest tree, so we - don't bother putting it inside a formal nix derivation check.) + (Note, this requires a full git history, not just the latest tree, so we don't bother putting it inside a formal nix derivation check.) diff --git a/subprojects/proto/v1/nix-support.proto b/subprojects/proto/v1/nix-support.proto new file mode 100644 index 000000000..8c312526c --- /dev/null +++ b/subprojects/proto/v1/nix-support.proto @@ -0,0 +1,34 @@ +// Message formats for the hydra-relevant information contained in +// the $output/nix-support directory. + +syntax = "proto3"; + +package runner.v1; + +import "v1/store.proto"; + +message BuildProduct { + nix.store.v1.RelativeStorePath path = 1; + string default_path = 2; + + string type = 3; + string subtype = 4; + string name = 5; + + bool is_regular = 6; + + optional string sha256hash = 7; + optional uint64 file_size = 8; +} + +message BuildMetric { + optional string unit = 1; + double value = 2; +} + +message NixSupport { + bool failed = 1; + optional string hydra_release_name = 2; + repeated BuildProduct products = 3; + map metrics = 4; +} diff --git a/subprojects/proto/v1/store.proto b/subprojects/proto/v1/store.proto new file mode 100644 index 000000000..caa7781b8 --- /dev/null +++ b/subprojects/proto/v1/store.proto @@ -0,0 +1,79 @@ +syntax = "proto3"; + +package nix.store.v1; + +message StorePath { + string path = 1; +} + +message StorePaths { + repeated StorePath paths = 1; +} + +message RelativeStorePath { + StorePath store_path = 1; + string sub_path = 2; +} + +message Hash { + enum Algorithm { + SHA256 = 0; + SHA512 = 1; + SHA1 = 2; + MD5 = 3; + BLAKE3 = 4; + } + Algorithm algorithm = 1; + bytes digest = 2; +} + +message ContentAddress { + enum Method { + TEXT = 0; + FLAT = 1; + NIX_ARCHIVE = 2; + } + Method method = 1; + Hash hash = 2; +} + +// Mirrors harmonia_store_core::store_path::ContentAddressMethodAlgorithm. +message ContentAddressMethodAlgorithm { + ContentAddress.Method method = 1; + Hash.Algorithm algorithm = 2; +} + +message Signature { + string key_name = 1; + string sig = 2; +} + +message UnkeyedValidPathInfo { + optional StorePath deriver = 1; + Hash nar_hash = 2; + repeated StorePath references = 3; + optional int64 registration_time = 4; + uint64 nar_size = 5; + bool ultimate = 6; + repeated Signature signatures = 7; + optional ContentAddress ca = 8; + string store_dir = 9; +} + +message ValidPathInfo { + StorePath path = 1; + UnkeyedValidPathInfo info = 2; +} + +message UnkeyedNarInfo { + UnkeyedValidPathInfo info = 1; + string url = 2; + string compression = 3; + optional Hash download_hash = 4; + optional uint64 download_size = 5; +} + +message NarInfo { + StorePath path = 1; + UnkeyedNarInfo info = 2; +} diff --git a/subprojects/proto/v1/store/derivation.proto b/subprojects/proto/v1/store/derivation.proto new file mode 100644 index 000000000..5034cda4c --- /dev/null +++ b/subprojects/proto/v1/store/derivation.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package nix.store.derivation.v1; + +import "v1/store.proto"; + +// Mirrors harmonia_store_core::derivation::DerivationOutput. +message Output { + oneof output { + // InputAddressed: statically-known output path. + nix.store.v1.StorePath input_addressed = 1; + // CAFixed: content-addressed fixed-output (e.g. fetchurl). + nix.store.v1.ContentAddress ca_fixed = 2; + // Deferred: no statically-known output path. + Unit deferred = 3; + // CAFloating: content-addressed with floating output path. + nix.store.v1.ContentAddressMethodAlgorithm ca_floating = 4; + // Impure: like CAFloating but not reproducible. + nix.store.v1.ContentAddressMethodAlgorithm impure = 5; + } +} + +// A force-resolved BasicDerivation ready to be built. +// All inputs are concrete store paths (no Built references). +// +// Mirrors harmonia_store_core::derivation::BasicDerivation. +message Basic { + string name = 1; + map outputs = 2; + repeated nix.store.v1.StorePath inputs = 3; + bytes platform = 4; + bytes builder = 5; + repeated bytes args = 6; + map env = 7; + // Optional structured attributes (JSON-encoded). + optional string structured_attrs = 8; +} + +// Empty message used as a unit type. +message Unit {} diff --git a/subprojects/proto/v1/streaming.proto b/subprojects/proto/v1/streaming.proto index c94986180..2c5b524bf 100644 --- a/subprojects/proto/v1/streaming.proto +++ b/subprojects/proto/v1/streaming.proto @@ -2,6 +2,10 @@ syntax = "proto3"; package runner.v1; +import "v1/store.proto"; +import "v1/store/derivation.proto"; +import "v1/nix-support.proto"; + option java_multiple_files = true; option java_outer_classname = "RunnerProto"; option java_package = "io.grpc.hydra.runner"; @@ -10,13 +14,12 @@ service RunnerService { rpc CheckVersion(VersionCheckRequest) returns (VersionCheckResponse) {} rpc OpenTunnel(stream BuilderRequest) returns (stream RunnerRequest) {} rpc BuildLog(stream LogChunk) returns (Empty) {} - rpc BuildResult(stream NarData) returns (Empty) {} + rpc BuildResult(stream AddToStoreRequest) returns (Empty) {} rpc BuildStepUpdate(StepUpdate) returns (Empty) {} rpc CompleteBuild(BuildResultInfo) returns (Empty) {} - rpc FetchDrvRequisites(FetchRequisitesRequest) returns (DrvRequisitesMessage) {} - rpc HasPath(StorePath) returns (HasPathResponse) {} - rpc StreamFile(StorePath) returns (stream NarData) {} - rpc StreamFiles(StorePaths) returns (stream NarData) {} + rpc FetchRequisites(nix.store.v1.StorePaths) returns (RequisitesResponse) {} + rpc HasPath(nix.store.v1.StorePath) returns (HasPathResponse) {} + rpc FetchPaths(nix.store.v1.StorePaths) returns (stream AddToStoreRequest) {} rpc RequestPresignedUrl(PresignedUrlRequest) returns (PresignedUrlResponse) {} rpc NotifyPresignedUploadComplete(PresignedUploadComplete) returns (Empty) {} } @@ -124,7 +127,7 @@ message PresignedUploadOpts { message BuildMessage { string build_id = 1; // UUIDv4 - StorePath drv = 2; + nix.store.v1.StorePath drv = 2; reserved 3; uint64 max_log_size = 4; int32 max_silent_time = 5; @@ -132,10 +135,16 @@ message BuildMessage { optional PresignedUploadOpts presigned_url_opts = 7; // None if presigned url upload is disabled // bool is_deterministic = 5; // bool enforce_determinism = 6; + + // Force-resolved derivation. The builder uses this with + // buildDerivation via the daemon protocol, rather than fetching + // the drv closure from the queue runner. All inputDrvs have been + // resolved to concrete output store paths in inputSrcs. + optional nix.store.derivation.v1.Basic resolved_drv = 8; } -message DrvRequisitesMessage { - repeated StorePath requisites = 1; +message RequisitesResponse { + repeated nix.store.v1.StorePath requisites = 1; } message AbortMessage { @@ -143,76 +152,38 @@ message AbortMessage { } message LogChunk { - StorePath drv = 1; + nix.store.v1.StorePath drv = 1; bytes data = 2; } -message FetchRequisitesRequest { - StorePath path = 1; - bool include_outputs = 2; -} - -message StorePath { - string path = 1; -} - -message StorePaths { - repeated StorePath paths = 1; -} - message HasPathResponse { bool has_path = 1; } -message NarData { - bytes chunk = 1; -} - -message OutputNameOnly { - string name = 1; -} - -message OutputWithPath { - string name = 1; - StorePath path = 2; - uint64 closure_size = 3; - uint64 nar_size = 4; - string nar_hash = 5; -} - -message Output { - oneof output { - OutputNameOnly nameonly = 1; - OutputWithPath withpath = 2; +message AddToStoreRequest { + // Rules below for streaming (i.e. `stream AddToStoreRequest`). Pity we cannot + // encode this in gRPC types. + oneof content { + // First send one of these. The header includes NAR sizes so the + // receiver knows how many uncompressed bytes to expect per path. + AddToStoreHeader header = 1; + // Then send as many of these as are needed. These are for a single + // zstd-compressed NAR data for all paths concatenated in header + // order. + bytes nar_chunk = 2; } } -message BuildMetric { - string path = 1; - string name = 2; - optional string unit = 3; - double value = 4; -} - -message BuildProduct { - string path = 1; - string default_path = 2; - - string type = 3; - string subtype = 4; - string name = 5; - - bool is_regular = 6; - - optional string sha256hash = 7; - optional uint64 file_size = 8; +message AddToStoreHeader { + repeated nix.store.v1.ValidPathInfo path_infos = 1; } -message NixSupport { - bool failed = 1; - optional string hydra_release_name = 2; - repeated BuildMetric metrics = 3; - repeated BuildProduct products = 4; +message OutputInfo { + nix.store.v1.StorePath path = 1; + uint64 closure_size = 2; + uint64 nar_size = 3; + nix.store.v1.Hash nar_hash = 4; + optional NixSupport nixSupport = 5; } enum StepStatus { @@ -248,13 +219,12 @@ message BuildResultInfo { uint64 build_time_ms = 4; uint64 upload_time_ms = 5; BuildResultState result_state = 6; - NixSupport nix_support = 7; - repeated Output outputs = 8; + map outputInfos = 7; } message PresignedNarRequest { string store_path = 1; - string nar_hash = 2; + nix.store.v1.Hash nar_hash = 2; repeated string debug_info_build_ids = 3; } @@ -286,14 +256,5 @@ message PresignedUrlResponse { message PresignedUploadComplete { string build_id = 1; // UUIDv4 string machine_id = 2; // UUIDv4 - string store_path = 3; - string url = 4; - string compression = 5; - string file_hash = 6; - uint64 file_size = 7; - string nar_hash = 8; - uint64 nar_size = 9; - repeated string references = 10; - optional string deriver = 11; - optional string ca = 12; + nix.store.v1.NarInfo nar_info = 13; }