diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 316f10ba..642bfce4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,9 @@ jobs: - name: Check that legal:check passes hopin's legal files at the tag it runs, names file and line of a broken one and quotes no text run: mise exec -- tests/legal/legal_check_names_file_and_line_and_quotes_no_text.sh + - name: Check that a failed LiteLLM write stops the rest and names what was made, and that no key value reaches an argument or an output + run: mise exec -- tests/litellm/failed_write_names_what_is_done.sh + - name: Check that the Renovate report check fails naming a missing report, a report with no dependency and each missing or skipped pin run: tests/renovate/report_check_names_each_missing_pin.sh @@ -391,6 +394,14 @@ jobs: LITELLM_NAMESPACE: litellm run: tests/litellm/expire_deletes_only_what_is_past_90_days.sh + # The step suspends the Kustomization litellm, which would otherwise restore the table it changes. + - name: P9c cells + if: matrix.part == 'postgres' + env: + LITELLM_NAMESPACE: litellm + APP_NAMESPACE: hopin + run: tests/litellm/keys_table_drives_budgets_and_the_app_key.sh + - name: Assert that /impressum reaches a Pod beside cloudflared through Traefik with the committed bytes and header and everything else gets 404 if: matrix.part == 'core' env: diff --git a/AGENTS.md b/AGENTS.md index 02ab9b3d..6828e78a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -366,6 +366,20 @@ Job made from the CronJob: it prints one deletion in each count and leaves the fresh rows and the idle customers with a budget or with spend, a second run prints zeros, and the first log holds no planted value; its Pod ran in a Namespace enforcing `restricted` with its Postgres's image. + The step "P9c cells", in the part `postgres`, suspends the Kustomization `litellm` and resumes it on every exit, removing + its keys, person budget and Secret key: `mise run litellm:keys --plan` lists the differences between the table and two + keys made with the master key, exits 2 and changes no row; the run sets the numbers, a second run prints 0 changes and a + raised budget changes that field alone and keeps `budget_reset_at`; the person budget takes the table's number; a row + without a key is named and ends 1 after the other rows were made equal, with the models and routes LiteLLM and the + front's rules name; `litellm:key hopin` makes the row from a generated value in the Secret `hopin`, a client Pod in + `hopin`, which reads the header line from a mounted Secret through `curl --header @file`, gets 200 from the front's `GET /v1/models` with it, a second run prints `unchanged`, and a changed + value, one without `sk-`, a missing key and an app outside the table are refused and change no row; no output holds a + value, the master key or the value's sha256. The job `checks` runs `tests/litellm/failed_write_names_what_is_done.sh` + against a fake kubectl and curl: a write that fails midway stops the rest, names what was made, the one that failed, + also when its answer was lost, and what was not made, and prints no reply body; a budget-only change sends no period; + a table without a number for the person budget or a row's limits, an absent revision, one that changes while the table + is read and a value with a newline are refused before any write; and the value reaches curl's stdin and no argument + or output, a run under `bash -x` included. - `clusters/pc/flux-system/`: Flux's own components and its sync to this repository. `gotk-components.yaml` is generated by `mise run flux:export` for the components `FLUX_COMPONENTS` in `mise.toml` lists, which hold no Helm reconciler (every chart is rendered into Git); it is not edited by hand, and `cluster` fails when a regeneration differs. @@ -480,10 +494,25 @@ layer's Postgres, which a replacement copies, and it prints the five counts by name, never a row; a first `select 1`, tried up to 30 times a second apart inside the Pod since a fresh Pod can be refused until the NetworkPolicy controller admits its address, shows psql's stderr, so a failed connection names itself in the log, while the statement's stderr is discarded because - psql quotes row values. `mise exec -- tests/litellm/key_list_count_equals_key_table_rows.sh ` is the operator's, since it - reads the master key: through the PC's k3s and a port-forward it prints the keys LiteLLM's `/key/list` counts and the + psql quotes row values. `mise exec -- tests/litellm/key_list_count_equals_key_table_rows.sh ` + reads the master key from the cluster's Secret: through the PC's k3s and a port-forward it prints the keys LiteLLM's `/key/list` counts and the key table's rows, those of the dashboard's UI session (which `/key/list` omits) and the rest, then `equal` or `DIFFERS` for the count against the rest; each step runs under a deadline and names itself when it fails. + `keys.yaml` is the table of the person budget and each app's budget and parallel limit; the ConfigMap `litellm-keys` + carries it, without a hash suffix, as no pod reads it. `mise run litellm:keys [--plan]` makes the person budget (the id + of `max_end_user_budget_id`) and each row's key equal to it, sending only the fields that differ, so a pure budget + change keeps the period's reset; it prints the revision Flux applied to the table it read, which it refuses when absent or changed during the read, one + line per field with old and new, and a count; it refuses a table whose person budget or a row's budget or parallel + limit is no number, naming the row; a row without a key is named with `run litellm:key ` and ends 1 after the + rest; a failed write stops the rest and names what was made, the failed write, whose outcome is unknown when its + answer was lost, and what was not made; `--plan` writes nothing and exits 0 with no change, 2 with changes, 1 on a + failure. `mise run litellm:key ` registers key `litellm_key` of Secret `` in namespace `` (one printable + line without a trailing newline, `sk-` and at least 16 characters) as the row's key, prints `unchanged` when the stored hash equals the value's, + and refuses another value under the alias, changing nothing; a key's new value goes to `platform-secrets` first. Both + run through `scripts/litellm_keys.sh` with a port-forward and the master key: a value and the master key travel from + kubectl's stdout through shell variables to curl's stdin, never in an argument, a file or a child's environment, and are + never printed; the reply of `/key/generate` goes nowhere; a failure names the step and the status, never a body. The + rpm and tpm limits are not in the table and stay as they are. - `infrastructure/legal/`: the imprint at `hallucinai.de/impressum`, nginx's unprivileged image (a vendor entry) in a Namespace that enforces Pod Security `restricted`, one pod with `Recreate` and a read-only root. `built/impressum.html` is hal's `legal/frame/impressum.html` byte for byte, which holds no template field, and `default.conf` serves only diff --git a/infrastructure/litellm/keys.yaml b/infrastructure/litellm/keys.yaml new file mode 100644 index 00000000..222b6f1c --- /dev/null +++ b/infrastructure/litellm/keys.yaml @@ -0,0 +1,16 @@ +# The budgets and limits LiteLLM holds for the apps' keys and the person budget, applied by `mise run litellm:keys` +# (`--plan` lists the differences first); `mise run litellm:key ` registers an app's key from its Secret. One +# row per app, named like the app's namespace and Secret. The person budget's id is `max_end_user_budget_id` of +# litellm-config.yaml, the routes are those of front-route.yml and the models those LiteLLM lists. Amounts are dollars. +period: 1mo +person_budget: 2 +apps: + hopin: + budget: 10 + parallel: 5 + presentator: + budget: 10 + parallel: 5 + songmaker: + budget: 10 + parallel: 5 diff --git a/infrastructure/litellm/kustomization.yaml b/infrastructure/litellm/kustomization.yaml index 287c1654..0757b3f5 100644 --- a/infrastructure/litellm/kustomization.yaml +++ b/infrastructure/litellm/kustomization.yaml @@ -1,6 +1,6 @@ # LiteLLM, its front and its egress, beside its Postgres from the base. LiteLLM reads its file and each Traefik its # static file once, at start, while each Traefik watches its dynamic directory; so each ConfigMap's name carries its -# content's hash and a change rolls the pod. +# content's hash and a change rolls the pod. The ConfigMap litellm-keys is the exception: `mise run litellm:keys` reads it. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: litellm @@ -32,6 +32,12 @@ configMapGenerator: - name: litellm-expire files: - expire.sql + # No pod reads the table; the commands read it from the cluster, so its name carries no hash. + - name: litellm-keys + files: + - keys.yaml + options: + disableNameSuffixHash: true - name: agent-egress files: - traefik.yml=egress-traefik.yml diff --git a/mise.toml b/mise.toml index eede2a89..95d5f4ef 100644 --- a/mise.toml +++ b/mise.toml @@ -853,6 +853,16 @@ echo "keycloak:admin: open $admin_url/admin/master/console/ in a private window; exec kubectl --namespace keycloak port-forward --address 127.0.0.1 service/keycloak "${BASH_REMATCH[1]}:http" ''' +[tasks."litellm:keys"] +description = "Make LiteLLM's person budget and the app keys' limits equal to infrastructure/litellm/keys.yaml; --plan only lists the differences (exit 2 with changes)" +quiet = true +run = "scripts/litellm_keys.sh keys" + +[tasks."litellm:key"] +description = "Register the litellm_key of Secret as the key of the row in infrastructure/litellm/keys.yaml" +quiet = true +run = "scripts/litellm_keys.sh key" + [tasks."gatus:open"] description = "Forward Gatus's status page to this PC's loopback and print its address" quiet = true diff --git a/scripts/litellm_keys.sh b/scripts/litellm_keys.sh new file mode 100755 index 00000000..8dfd0040 --- /dev/null +++ b/scripts/litellm_keys.sh @@ -0,0 +1,344 @@ +#!/usr/bin/env bash +# The commands behind `mise run litellm:keys` and `mise run litellm:key`: they make the budgets and the app keys of +# LiteLLM in namespace litellm equal to the table infrastructure/litellm/keys.yaml, which the ConfigMap litellm-keys +# carries. +# scripts/litellm_keys.sh keys [--plan] sends only the fields that differ; --plan writes nothing and exits 0 with no +# change, 2 with changes, 1 on a failure or a table row without a key +# scripts/litellm_keys.sh key registers the value of key litellm_key in Secret of namespace as +# the key of the table's row +# A key's value and the master key travel from kubectl's stdout through shell variables to curl's stdin, never into an +# argument, a URL, a file or a child's environment, and are never printed. A failure names the step and the HTTP +# status, never a reply body. +# Tracing would print every decoded value, so it ends before any is read, without a line of its own. +{ set +x; } 2>/dev/null +set -euo pipefail +# [!-~] must mean printable ASCII. +export LC_ALL=C + +namespace=litellm +export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}" +# A kubectl call ends past the deadline, as a hung cluster would otherwise hold the command forever. +deadline=2m +# LiteLLM refuses a chosen key value that lacks this prefix or is shorter. +key_prefix=sk- +key_minimum_length=16 + +command_name="" +fail() { + echo "litellm:${command_name:-keys}: $1" >&2 + exit 1 +} +# bounded ...: runs the command under the deadline; fails naming the step when it fails or runs out. +bounded() { + local step="$1" status=0 + shift + timeout "$deadline" "$@" 2>/dev/null || status=$? + [ "$status" != 124 ] || fail "$step did not end within $deadline, the local deadline" + [ "$status" = 0 ] || fail "$step failed" +} +# in_cluster ...: kubectl's stdout; its stderr stays hidden, since a message may quote a value. +in_cluster() { + local step="$1" + shift + bounded "$step" kubectl "$@" +} +# json ...: yq over the last reply. +json() { printf '%s' "$reply" | yq --input-format json "$@"; } + +scratch="$(mktemp -d)" +forward="" +trap '[ -z "$forward" ] || kill "$forward" 2>/dev/null || true; rm -rf "$scratch"' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# decode : sets decoded to the exact bytes, through a pipe so that no file holds them; the sentinel keeps the +# command substitution from stripping a trailing newline, which validation must see. +decoded="" +decode() { + decoded="$(printf '%s' "$1" | base64 --decode && printf x)" || return 1 + decoded="${decoded%x}" +} +# quote_inner : the text as the inside of a double-quoted curl config value or JSON string; the callers hold only +# printable ASCII. +quote_inner() { + local text="${1//\\/\\\\}" + printf '%s' "${text//\"/\\\"}" +} + +reply="" status="" failure="" +# request [ [discard]]: a GET without a body, a POST with one, authorized by the master key on +# curl's stdin; sets status and reply, with discard the reply goes nowhere. Returns 1 with failure naming the step and +# the status when the status is not 200. +request() { + local step="$1" path="$2" body="${3-}" config result + local -a sink=() + config="header = \"Authorization: Bearer $(quote_inner "$master_key")\"" + if [ -n "$body" ]; then + config+=$'\nheader = "Content-Type: application/json"\ndata = "'"$(quote_inner "$body")"'"' + fi + [ -z "${4-}" ] || sink=(--output /dev/null) + result="$(printf '%s\n' "$config" | curl --silent --max-time 60 --config - "${sink[@]}" \ + --write-out '\n%{http_code}' "http://127.0.0.1:$local_port$path")" || result=$'\n000' + status="${result##*$'\n'}" + reply="${result%"$status"}" + [ "$status" = 200 ] || { + failure="$step answered with status $status, not 200" + return 1 + } +} +# ask ...: request, ending the command on a failure. +ask() { request "$@" || fail "$failure"; } + +# connect: reads the master key and starts the port-forward to LiteLLM; sets master_key and local_port. +connect() { + local secret_data port + secret_data="$(in_cluster "reading the Secret litellm" --namespace "$namespace" get secret litellm \ + -o jsonpath='{.data.master_key}')" + decode "$secret_data" || fail "the Secret litellm gives no readable master_key" + master_key="$decoded" + [ -n "$master_key" ] || fail "the Secret litellm holds an empty master_key" + port="$(in_cluster "reading the Service litellm" --namespace "$namespace" get service litellm \ + -o jsonpath='{.spec.ports[0].port}')" + timeout 10m kubectl --namespace "$namespace" port-forward service/litellm ":$port" >"$scratch/forward.log" 2>&1 & + forward=$! + local_port="" + for _ in $(seq 30); do + local_port="$(sed -n 's/^Forwarding from 127\.0\.0\.1:\([0-9]*\) .*/\1/p' "$scratch/forward.log" | head -n 1)" + [ -z "$local_port" ] || break + kill -0 "$forward" 2>/dev/null || fail "kubectl port-forward to the Service litellm ended" + sleep 1 + done + [ -n "$local_port" ] || fail "kubectl port-forward to the Service litellm did not start within 30 seconds" +} + +# applied_revision: the Git revision Flux applied to the layer. +applied_revision() { + in_cluster "reading the Kustomization litellm" --namespace flux-system get kustomization litellm \ + -o jsonpath='{.status.lastAppliedRevision}' +} +# is_number : whether the tag names a number. +is_number() { [ "$1" = '!!int' ] || [ "$1" = '!!float' ]; } +# check_table: refuses a table whose person budget or any row's budget or parallel limit is no number, whose apps are no +# map of rows or whose names are no namespace names, naming the row; sets app_names, the rows' names in order. +check_table() { + local rows name row_tag budget_tag parallel_tag + app_names=() + budget_tag="$(yq '.person_budget | tag' <<<"$table")" || fail "the table is no YAML" + is_number "$budget_tag" || fail "the table's person_budget is no number" + rows="$(yq '.apps | select(tag == "!!map") | to_entries | sort_by(.key) | .[] | .key + " " + (.value | tag) + " " + + ((.value | select(tag == "!!map") | .budget | tag) // "none") + " " + + ((.value | select(tag == "!!map") | .parallel | tag) // "none")' <<<"$table")" || fail "the table is no YAML" + [ -n "$rows" ] || fail "the table's apps are no map with a row" + while read -r name row_tag budget_tag parallel_tag; do + [[ $name =~ ^[a-z0-9-]+$ ]] || fail "the table names the app '$name', which is no namespace name" + [ "$row_tag" = '!!map' ] || fail "the table's row $name is no map" + is_number "$budget_tag" || fail "the table's row $name has no number budget" + [ "$parallel_tag" = '!!int' ] || fail "the table's row $name has no whole number parallel" + app_names+=("$name") + done <<<"$rows" +} + +# read_table: sets table, the ConfigMap litellm-keys' table, checked, with the Git revision Flux applied to it, which is +# refused when absent or changed while the table was read, and the settings every row shares from their one owners: +# period, budget_id (litellm-config.yaml), routes (front-route.yml) and models (as LiteLLM lists them). +read_table() { + local config_map config revision + revision="$(applied_revision)" + [ -n "$revision" ] || fail "the Kustomization litellm has applied no revision" + table="$(in_cluster "reading the ConfigMap litellm-keys" --namespace "$namespace" get configmap litellm-keys \ + -o jsonpath='{.data.keys\.yaml}')" + [ "$(applied_revision)" = "$revision" ] || fail "the Kustomization litellm applied another revision while the table was read" + echo "litellm:$command_name: table of revision $revision" + check_table + period="$(yq '.period' <<<"$table")" + [[ $period =~ ^[0-9]+[a-z]+$ ]] || fail "the table's period '$period' is not a LiteLLM duration" + config_map="$(in_cluster "reading the Deployment litellm" --namespace "$namespace" get deployment litellm \ + -o jsonpath='{.spec.template.spec.volumes[?(@.name=="config")].configMap.name}')" + config="$(in_cluster "reading LiteLLM's configuration" --namespace "$namespace" get configmap "$config_map" \ + -o jsonpath='{.data.config\.yaml}')" + budget_id="$(yq '.litellm_settings.max_end_user_budget_id' <<<"$config")" + [[ $budget_id =~ ^[A-Za-z0-9_-]+$ ]] || fail "LiteLLM's configuration names no person budget id" + config_map="$(in_cluster "reading the Deployment litellm-front" --namespace "$namespace" get deployment \ + litellm-front -o jsonpath='{.spec.template.spec.volumes[?(@.name=="routes")].configMap.name}')" + routes="$(in_cluster "reading the front's routes" --namespace "$namespace" get configmap "$config_map" \ + -o jsonpath='{.data.route\.yml}' | yq --output-format json --indent 0 \ + '[.http.routers[].rule | capture("Path[(]`(?P[^`]+)`[)]").path] | sort')" + ask "listing LiteLLM's models" /v1/models + models="$(json --output-format json --indent 0 '[.data[].id] | sort')" + [ "$models" != "[]" ] || fail "LiteLLM lists no model" +} + +# desired_key : the key fields the table row and the shared settings make, as JSON; the table is checked. +desired_key() { + APP="$1" PERIOD="$period" ROUTES="$routes" MODELS="$models" yq --output-format json --indent 0 ' + .apps[strenv(APP)] as $row | {"max_budget": $row.budget, "budget_duration": strenv(PERIOD), + "max_parallel_requests": $row.parallel, "models": (strenv(MODELS) | from_json), + "allowed_routes": (strenv(ROUTES) | from_json)}' <<<"$table" +} +# flat : one "field value" line per field, a list sorted and joined and a number as a number, so that two +# objects compare by their lines. +flat() { + printf '%s' "$1" | yq --input-format json 'to_entries | .[] | .key + " " + + ((select(.value | tag == "!!seq") | .value | map(tostring) | sort | join(",")) // (.value | tostring))' +} +# differences : sets change_fields to the JSON of the desired fields that differ +# and change_text to one line per difference, field old -> new. +differences() { + local desired="$1" found="$2" field value flat_desired flat_found="" have_value="" + local -A have=() + change_fields="" change_text="" + flat_desired="$(flat "$desired")" + [ "$found" = null ] || flat_found="$(flat "$found")" + while read -r field value; do + [ -z "$field" ] || have["$field"]="$value" + done <<<"$flat_found" + while read -r field value; do + have_value="${have[$field]-null}" + [ "$have_value" != "$value" ] || continue + change_fields+="${change_fields:+,}\"$field\":$(yq --input-format json --output-format json --indent 0 ".$field" <<<"$desired")" + change_text+="${change_text:+$'\n'}$field $have_value -> $value" + done <<<"$flat_desired" + change_fields="{$change_fields}" +} +# found_keys : sets key_count to the keys that hold the alias exactly and found_row to the first of them as JSON, or +# null. +found_keys() { + local holders + ask "listing the keys of $1" "/key/list?key_alias=$1&return_full_object=true&size=100" + holders="$(json --output-format json --indent 0 '[.keys[] | select(.key_alias == "'"$1"'")]')" + key_count="$(yq --input-format json 'length' <<<"$holders")" + found_row="$(yq --input-format json --output-format json --indent 0 '.[0]' <<<"$holders")" +} +# key_hash: the stored hash of the key found_keys found. +key_hash() { yq --input-format json '.token' <<<"$found_row"; } + +# Each change is one write: its kind, its subject and its body. The change in change_fields and change_text is added +# after differences, and its lines are printed and counted. +change_kinds=() change_subjects=() change_bodies=() +change_lines=0 +add_change() { + local kind="$1" subject="$2" body line + [ "$change_fields" != "{}" ] || return 0 + case "$kind" in + budget-new | budget-update) body="{\"budget_id\":\"$subject\",${change_fields#\{}" ;; + key-update) body="{\"key\":\"$(key_hash)\",${change_fields#\{}" ;; + esac + change_kinds+=("$kind") + change_subjects+=("$subject") + change_bodies+=("$body") + while IFS= read -r line; do + echo "litellm:keys: ${kind%%-*} $subject $line" + change_lines=$((change_lines + 1)) + done <<<"$change_text" +} + +keys_command() { + local plan=false app desired found kind path index not_made_index missing=0 made="" not_made="" + case "$*" in + "") ;; + --plan) plan=true ;; + *) fail "usage: scripts/litellm_keys.sh keys [--plan]" ;; + esac + connect + read_table + + # The person budget, then each app in name order. + desired="$(PERIOD="$period" yq --output-format json --indent 0 '{"max_budget": .person_budget, + "budget_duration": strenv(PERIOD)}' <<<"$table")" + ask "listing the budgets" /budget/list + found="$(json --output-format json --indent 0 '[.[] | select(.budget_id == "'"$budget_id"'")] | .[0]')" + differences "$desired" "$found" + if [ "$found" = null ]; then kind=budget-new; else kind=budget-update; fi + add_change "$kind" "$budget_id" + for app in "${app_names[@]}"; do + found_keys "$app" + if [ "$key_count" != 1 ]; then + if [ "$key_count" = 0 ]; then + echo "litellm:keys: key $app: none; run mise run litellm:key $app" + else + echo "litellm:keys: key $app: $key_count keys hold the alias, none is changed" + fi + missing=$((missing + 1)) + continue + fi + desired="$(desired_key "$app")" + differences "$desired" "$found_row" + add_change key-update "$app" + done + + if [ "$plan" = true ]; then + echo "litellm:keys: $change_lines changes planned, nothing written" + [ "$missing" = 0 ] || exit 1 + [ "$change_lines" = 0 ] || exit 2 + return 0 + fi + for index in "${!change_kinds[@]}"; do + case "${change_kinds[$index]}" in + budget-new) path=/budget/new ;; + budget-update) path=/budget/update ;; + key-update) path=/key/update ;; + esac + if ! request "writing ${change_kinds[$index]%%-*} ${change_subjects[$index]}" "$path" "${change_bodies[$index]}" discard; then + # The write may have reached LiteLLM before its answer was lost, so it is neither made nor not made. + for ((not_made_index = index + 1; not_made_index < ${#change_kinds[@]}; not_made_index++)); do + not_made+=" ${change_kinds[$not_made_index]%%-*}:${change_subjects[$not_made_index]}" + done + echo "litellm:keys: $failure; made:${made:- none}; failed: ${change_kinds[$index]%%-*}:${change_subjects[$index]};" \ + "not made:${not_made:- none}" >&2 + exit 1 + fi + made+=" ${change_kinds[$index]%%-*}:${change_subjects[$index]}" + done + echo "litellm:keys: $change_lines changes made" + [ "$missing" = 0 ] || exit 1 +} + +key_command() { + local app="${1-}" secret_data value value_hash desired body table_app has_row=false + [ "$#" = 1 ] || fail "usage: scripts/litellm_keys.sh key " + connect + read_table + for table_app in "${app_names[@]}"; do + [ "$table_app" != "$app" ] || has_row=true + done + [ "$has_row" = true ] || fail "$app has no row in the table" + secret_data="$(in_cluster "reading the Secret $app" --namespace "$app" get secret "$app" \ + -o jsonpath='{.data.litellm_key}')" + [ -n "$secret_data" ] || fail "the Secret $app has no litellm_key" + decode "$secret_data" || fail "the litellm_key of $app is no base64" + value="$decoded" + [[ $value == "$key_prefix"* && $value =~ ^[!-~]+$ ]] || + fail "the litellm_key of $app is not one printable line starting with $key_prefix" + [ "${#value}" -ge "$key_minimum_length" ] || fail "the litellm_key of $app is shorter than $key_minimum_length characters" + value_hash="$(printf '%s' "$value" | sha256sum)" + value_hash="${value_hash%% *}" + + found_keys "$app" + case "$key_count" in + 0) ;; + 1) + [ "$(key_hash)" = "$value_hash" ] || fail "another key value holds the alias $app; nothing changed" + echo "litellm:key: $app unchanged" + return 0 + ;; + *) fail "$key_count keys hold the alias $app; nothing changed" ;; + esac + desired="$(desired_key "$app")" + body="{\"key\":\"$(quote_inner "$value")\",\"key_alias\":\"$app\",${desired#\{}" + ask "making the key of $app" /key/generate "$body" discard + echo "litellm:key: $app made" +} + +case "${1-}" in + keys) + command_name=keys + shift + keys_command "$@" + ;; + key) + command_name=key + shift + key_command "$@" + ;; + *) fail "usage: scripts/litellm_keys.sh keys [--plan] | key " ;; +esac diff --git a/tests/litellm/failed_write_names_what_is_done.sh b/tests/litellm/failed_write_names_what_is_done.sh new file mode 100755 index 00000000..fc1636a3 --- /dev/null +++ b/tests/litellm/failed_write_names_what_is_done.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# Drive `mise run litellm:keys` and `mise run litellm:key` against a fake kubectl and a fake curl, for the outcomes the +# cluster cannot arrange: a write that fails midway stops the rest, names what was made, which one failed and what was +# not made, and prints no reply body, a lost answer included; a failed key registration names the step and the status; a +# budget-only change sends no period; a table that is no table of numbers, a revision that is absent or changes while the +# table is read and a value with a newline are refused before any write; and neither a key's value, the master key nor +# the value's sha256 reaches an argument or any output, tracing included, while the value does reach curl's stdin. +# Usage: mise exec -- tests/litellm/failed_write_names_what_is_done.sh +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=../failure.sh +source "$repository_root/tests/failure.sh" +work_directory="$(mktemp -d)" +trap 'rm -rf "$work_directory"' EXIT +fake_bin="$work_directory/bin" +mkdir "$fake_bin" +layer="$repository_root/infrastructure/litellm" +master_key=sk-fake-master-key-0001 +app_key=sk-fake-app-value-0123456789 +reply_body=REPLY-BODY-THAT-MUST-NOT-APPEAR + +# The fake answers what the task reads from the cluster; the shipped table, configuration and routes are its data. +cat >"$fake_bin/kubectl" <<'FAKE' +#!/usr/bin/env bash +set -euo pipefail +case " $* " in *" sk-"*) echo "fake kubectl: a key in the arguments" >&2; exit 1 ;; esac +case "$*" in + *"get secret litellm "*) printf '%s' "$FAKE_MASTER_KEY" | base64 --wrap=0 ;; + *"get service litellm "*) printf 4000 ;; + *port-forward*) echo "Forwarding from 127.0.0.1:4000 -> 4000"; exec sleep 600 ;; + *"get configmap litellm-keys"*) cat "$FAKE_LAYER/keys.yaml" ;; + *"get deployment litellm "*) printf litellm-config ;; + *"get configmap litellm-config"*) cat "$FAKE_LAYER/litellm-config.yaml" ;; + *"get deployment litellm-front"*) printf front-routes ;; + *"get configmap front-routes"*) cat "$FAKE_LAYER/front-route.yml" ;; + *"get kustomization litellm"*) + reads="$(cat "$FAKE_STATE/revision-reads" 2>/dev/null || echo 0)" + echo $((reads + 1)) >"$FAKE_STATE/revision-reads" + case "$FAKE_REVISION" in + absent) ;; + changing) printf 'main@sha1:read-%s' "$reads" ;; + *) printf 'main@sha1:fake' ;; + esac ;; + *"get secret hopin "*) printf '%s' "$FAKE_SECRET_VALUE" | base64 --wrap=0 ;; + *) echo "fake kubectl: unexpected $*" >&2; exit 1 ;; +esac +FAKE +# The fake LiteLLM lists one model and keeps what it is told in FAKE_STATE: the person budget, and for each app but hopin +# a key, which starts with fields that all differ from the table. A write named by FAKE_FAILING_WRITE (a path and a +# body marker) is answered with 500 and a body, or with FAKE_TRANSPORT_FAILURE after it was applied with a lost answer; +# every other write is applied and answered 200. It records each write's path, the body of the updates, and whether the +# key value reached its stdin. +cat >"$fake_bin/curl" <<'FAKE' +#!/usr/bin/env bash +set -euo pipefail +url="${*: -1}" +path="${url#http://127.0.0.1:4000}" +config="$(cat)" +for argument in "$@"; do + case "$argument" in *"$FAKE_MASTER_KEY"* | *"$FAKE_APP_KEY"*) echo "fake curl: a key in the arguments" >&2; exit 1 ;; esac +done +case "$config" in *"Bearer $FAKE_MASTER_KEY"*) ;; *) echo "fake curl: no master key on stdin" >&2; exit 1 ;; esac +case "$config" in *"$FAKE_APP_KEY"*) echo "app-key-on-stdin $path" >>"$FAKE_STATE/writes" ;; esac +json() { yq --input-format json --output-format json --indent 0 "$@"; } +status=200 reply='{}' +case "$path" in + /v1/models) reply='{"data":[{"id":"model-one"}]}' ;; + /budget/list) + reply='[]' + [ ! -f "$FAKE_STATE/budget.json" ] || reply="[$(cat "$FAKE_STATE/budget.json")]" ;; + "/key/list?key_alias=hopin"*) reply='{"keys":[]}' ;; + "/key/list?key_alias="*) + alias="${path#*key_alias=}" + alias="${alias%%&*}" + row="$FAKE_STATE/row-$alias.json" + [ -f "$row" ] || printf '{"key_alias":"%s","token":"hash-%s","max_budget":1,"budget_duration":"1d","max_parallel_requests":1,"models":[],"allowed_routes":[]}' \ + "$alias" "$alias" >"$row" + reply="{\"keys\":[$(cat "$row")]}" ;; + *) + echo "write $path" >>"$FAKE_STATE/writes" + data="$(grep '^data = ' <<<"$config")" + data="${data#data = \"}" + data="$(sed 's/\\"/"/g; s/\\\\/\\/g' <<<"${data%\"}")" + failing=false + [ -z "$FAKE_FAILING_WRITE" ] || case "$path $config" in *"$FAKE_FAILING_WRITE"*) failing=true ;; esac + if [ "$failing" = false ] || [ -n "$FAKE_TRANSPORT_FAILURE" ]; then + case "$path" in + /key/update) + echo "update $data" >>"$FAKE_STATE/bodies" + target="$(yq --input-format json '.key | sub("^hash-"; "")' <<<"$data")" + json -i ". * $data" "$FAKE_STATE/row-$target.json" ;; + /budget/new | /budget/update) + echo "budget $data" >>"$FAKE_STATE/bodies" + [ -f "$FAKE_STATE/budget.json" ] || echo '{}' >"$FAKE_STATE/budget.json" + json -i ". * $data" "$FAKE_STATE/budget.json" ;; + esac + fi + [ "$failing" = false ] || { [ -z "$FAKE_TRANSPORT_FAILURE" ] || exit 28; status=500 reply="$FAKE_REPLY_BODY"; } ;; +esac +case " $* " in *" --output /dev/null "*) reply="" ;; esac +printf '%s\n%s' "$reply" "$status" +FAKE +chmod +x "$fake_bin/kubectl" "$fake_bin/curl" +# mise puts its own kubectl before PATH and the tasks fall back to the PC's kubeconfig, so neither may be reachable. +export PATH="$fake_bin:$PATH" MISE_DISABLE_TOOLS=kubectl,flux,flux2 KUBECONFIG=/nonexistent +[ "$(command -v kubectl)" = "$fake_bin/kubectl" ] || fail "kubectl on PATH is not the fake" +[ "$(command -v curl)" = "$fake_bin/curl" ] || fail "curl on PATH is not the fake" +export FAKE_MASTER_KEY="$master_key" FAKE_APP_KEY="$app_key" FAKE_SECRET_VALUE="$app_key" FAKE_REPLY_BODY="$reply_body" +export FAKE_REVISION=applied FAKE_TRANSPORT_FAILURE="" + +budget_id="$(yq '.litellm_settings.max_end_user_budget_id' "$layer/litellm-config.yaml")" +# use_table []: the layer's files with the table reduced to the test's own rows and then edited, which the +# fake serves from now on. The rows sort as hopin, test-a, test-b, and hopin is the one without a key. +use_table() { + rm -rf "$work_directory/layer" + cp -r "$layer" "$work_directory/layer" + yq -i '.apps = {"hopin": {"budget": 10, "parallel": 5}, "test-a": {"budget": 10, "parallel": 5}, "test-b": {"budget": 10, "parallel": 5}}' \ + "$work_directory/layer/keys.yaml" + [ -z "${1-}" ] || yq -i "$1" "$work_directory/layer/keys.yaml" + export FAKE_LAYER="$work_directory/layer" +} +use_table + +runner=(mise run) +# run_task ...: runs the task, which must end as +# expected, and sets status, output and writes; a case name used again keeps its fake LiteLLM's state. +run_task() { + local case_name="$1" expected="$2" secret + export FAKE_FAILING_WRITE="$3" FAKE_STATE="$work_directory/state-$1" + shift 3 + mkdir -p "$FAKE_STATE" + : >"$FAKE_STATE/writes" + : >"$FAKE_STATE/bodies" + rm -f "$FAKE_STATE/revision-reads" + status=0 + output="$(cd "$repository_root" && "${runner[@]}" "$@" 2>&1)" || status=$? + writes="$(cat "$FAKE_STATE/writes")" + # A value the task must never print: the keys, and the sha256 of the app key. + value_hash="$(printf '%s' "$app_key" | sha256sum)" + for secret in "$master_key" "$app_key" "$reply_body" "${value_hash%% *}"; do + case "$output" in *"$secret"*) fail "$case_name: the output holds a value or a reply body" ;; esac + done + [ "$status" = "$expected" ] || fail "$case_name: exit $status, not $expected: $output" +} + +run_task failed-midway 1 /key/update litellm:keys +grep -q "writing key test-a answered with status 500, not 200; made: budget:$budget_id; failed: key:test-a; not made: key:test-b\$" \ + <<<"$output" || fail "failed-midway: the output does not name what was made, what failed and what was not: $output" +grep -q '^litellm:keys: table of revision main@sha1:fake$' <<<"$output" || fail "failed-midway: no revision: $output" +[ "$(grep -c '^write ' <<<"$writes")" = 2 ] || + fail "failed-midway: a write ran after the failed one, or the made ones did not run: $writes" +echo "a write that fails midway stops the rest and names what was made, which one failed and what was not, with no reply body" + +FAKE_TRANSPORT_FAILURE=1 run_task lost-answer 1 hash-test-a litellm:keys +grep -q "writing key test-a answered with status 000, not 200; made: budget:$budget_id; failed: key:test-a; not made: key:test-b\$" \ + <<<"$output" || fail "lost-answer: the write that may have happened is not named as failed: $output" +FAKE_TRANSPORT_FAILURE=1 run_task lost-last-answer 1 hash-test-b litellm:keys +grep -q 'failed: key:test-b; not made: none$' <<<"$output" || + fail "lost-last-answer: the last write is not named as failed with nothing left: $output" +echo "a write whose answer is lost is named as failed, apart from the ones never tried" + +run_task failed-registration 1 /key/generate litellm:key hopin +grep -q 'making the key of hopin answered with status 500, not 200' <<<"$output" || + fail "failed-registration: the output does not name the step and the status: $output" +grep -q '^litellm:key: table of revision main@sha1:fake$' <<<"$output" || fail "failed-registration: no revision: $output" +[ "$writes" = $'app-key-on-stdin /key/generate\nwrite /key/generate' ] || + fail "failed-registration: the value did not reach curl's stdin in one write: $writes" +echo "a failed registration names its step and status, and the value reaches only curl's stdin" + +runner=(bash -x "$repository_root/scripts/litellm_keys.sh") +run_task traced-registration 1 /key/generate key hopin +[ "$writes" = $'app-key-on-stdin /key/generate\nwrite /key/generate' ] || + fail "traced-registration: the value did not reach curl's stdin in one write: $writes" +runner=(mise run) +echo "a script run under bash -x prints no value" + +# A value that is not one printable line is refused before any write, a trailing newline included. +for value in $'sk-fake-app-value-0123456789\n' $'sk-fake-app-value\n0123456789' $'sk-fake-app-value-0123456789\n\n'; do + FAKE_SECRET_VALUE="$value" run_task newline-value 1 "" litellm:key hopin + grep -q 'is not one printable line starting with sk-' <<<"$output" || fail "newline-value: the value was not refused as such: $output" + [ -z "$writes" ] || fail "newline-value: a write ran: $writes" +done +echo "a value with a newline, a trailing one included, is refused before any write" + +# A table that is no table of numbers is refused, naming the row, before any write. +table_cases=( + 'no-person-budget~del(.person_budget)~the table.s person_budget is no number' + 'budget-typo~del(.apps.hopin.budget) | .apps.hopin.buget = 12~row hopin has no number budget' + 'budget-text~.apps.hopin.budget = "ten"~row hopin has no number budget' + 'parallel-missing~del(.apps.hopin.parallel)~row hopin has no whole number parallel' + 'row-scalar~.apps.hopin = 5~row hopin is no map' + 'apps-null~.apps = null~apps are no map with a row' + 'apps-number~.apps = 5~apps are no map with a row' + 'apps-empty~.apps = {}~apps are no map with a row' + 'bad-name~.apps.Bad_Name = .apps.hopin~which is no namespace name' +) +for table_case in "${table_cases[@]}"; do + IFS='~' read -r case_name edit expected <<<"$table_case" + for task in "litellm:keys" "litellm:key hopin"; do + use_table "$edit" + # shellcheck disable=SC2086 # the task and its argument are words + run_task "$case_name" 1 "" $task + grep -q "$expected" <<<"$output" || fail "$case_name: $task did not refuse the table as '$expected': $output" + [ -z "$writes" ] || fail "$case_name: $task wrote: $writes" + done +done +use_table +echo "a table without a number for the person budget or a row's limits, or without rows, is refused naming the row, with no write" + +for revision in absent changing; do + FAKE_REVISION="$revision" run_task "revision-$revision" 1 "" litellm:keys + grep -q 'litellm has applied no revision\|applied another revision while the table was read' <<<"$output" || + fail "revision-$revision: the revision was not refused as such: $output" + [ -z "$writes" ] || fail "revision-$revision: a write ran: $writes" +done +echo "an absent revision, or one that changes while the table is read, is refused before any write" + +# A budget-only change sends the budget alone, since a resent period would move the reset. +use_table 'del(.apps.hopin)' +run_task budget-only 0 "" litellm:keys +run_task budget-only 0 "" litellm:keys --plan +grep -q ' 0 changes planned' <<<"$output" || fail "budget-only: a second plan differs: $output" +use_table 'del(.apps.hopin) | .apps."test-a".budget += 1' +run_task budget-only 0 "" litellm:keys +[ "$(grep -c '^update ' "$FAKE_STATE/bodies")" = 1 ] || fail "budget-only: not one update: $(cat "$FAKE_STATE/bodies")" +grep -q '^update .*"max_budget"' "$FAKE_STATE/bodies" || fail "budget-only: the update holds no budget: $(cat "$FAKE_STATE/bodies")" +! grep -q 'budget_duration\|"models"\|allowed_routes\|max_parallel' "$FAKE_STATE/bodies" || + fail "budget-only: the update holds more than the budget: $(cat "$FAKE_STATE/bodies")" +use_table +echo "a changed budget alone sends the budget and no period" diff --git a/tests/litellm/key_list_count_equals_key_table_rows.sh b/tests/litellm/key_list_count_equals_key_table_rows.sh index a5be5d8b..713977e4 100755 --- a/tests/litellm/key_list_count_equals_key_table_rows.sh +++ b/tests/litellm/key_list_count_equals_key_table_rows.sh @@ -5,7 +5,7 @@ # total it counts, then equal or DIFFERS, never a key; exits 1 when they differ or a step fails, naming the step and, # when the local deadline ended it, that deadline. set -euo pipefail -usage="usage: $0 : the operator's to run, since it reads LiteLLM's master key from the Secret litellm" +usage="usage: $0 : reads LiteLLM's master key from the Secret litellm" [ "$#" = 1 ] || { echo "$usage" >&2 exit 2 diff --git a/tests/litellm/keys_table_drives_budgets_and_the_app_key.sh b/tests/litellm/keys_table_drives_budgets_and_the_app_key.sh new file mode 100755 index 00000000..53f5c738 --- /dev/null +++ b/tests/litellm/keys_table_drives_budgets_and_the_app_key.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# In k3d: the table in the ConfigMap litellm-keys drives LiteLLM's person budget and the app keys through +# `mise run litellm:keys` and `mise run litellm:key`: --plan lists the differences, exits 2 and changes no row; a row +# without a key is named and the run ends 1; the numbers reach a key the command did not make and a second run changes +# nothing; a changed budget changes only that field and keeps the period's reset; the person budget takes its declared +# number under the id LiteLLM's configuration names; litellm:key registers the value of the app's Secret so that it +# passes the front, a second run reports unchanged, and a changed value, one without sk-, a missing key and an app +# outside the table are refused and change no row; no output holds a value, the master key or the value's sha256. The +# rows carry the models LiteLLM lists and the routes the front's rules name, read from their owners. +# The Kustomization litellm is suspended meanwhile, so that it does not restore the table, and resumed on every exit. +set -euo pipefail +# No value of this cell reaches a trace. +{ set +x; } 2>/dev/null +# shellcheck source=../failure.sh +source "$(dirname "${BASH_SOURCE[0]}")/../failure.sh" +# The tasks fall back to the PC's kubeconfig, so the job's cluster is named for them. +export KUBECONFIG="$HOME/.kube/config" +namespace="$LITELLM_NAMESPACE" +app_namespace="$APP_NAMESPACE" +aliases=(hopin presentator songmaker) +forward="" local_port="" master_key="" +# admin []: the master key's call to LiteLLM, which prints the reply; the key goes through curl's stdin. +admin() { + local body=() + [ -z "${2-}" ] || body=(--data "$2") + printf 'header = "Authorization: Bearer %s"\n' "$master_key" | + curl --silent --show-error --fail-with-body --max-time 30 --config - \ + --header 'Content-Type: application/json' "${body[@]}" "http://127.0.0.1:$local_port$1" +} +# attempt ...: a cleanup step; one that fails is named and fails the cell after the rest ran. +attempt() { + "$@" || { + echo "::error::the cleanup step '$1 ${2-}' failed" + deletion_failures=$((deletion_failures + 1)) + } +} +clear_secret_key() { + kubectl --namespace "$app_namespace" patch secret "$app_namespace" --type merge --patch '{"data":{"litellm_key":null}}' >/dev/null +} +# admin_discard []: admin without its reply. +admin_discard() { admin "$@" >/dev/null; } +set_table() { + kubectl --namespace "$namespace" patch configmap litellm-keys --type merge \ + --patch "$(TABLE="$1" yq --null-input --output-format json --indent 0 '{"data": {"keys.yaml": strenv(TABLE)}}')" >/dev/null +} +# Removes what the cell added on every exit: its keys, the person budget, the Secret's key and the table, then stops the +# port-forward the deletions need and resumes the Kustomization, which restores what Flux owns. +cleanup() { + local cell_status=$? alias + delete_object --namespace "$app_namespace" delete pod keys-client --ignore-not-found --wait --timeout=2m + delete_object --namespace "$app_namespace" delete secret keys-client-header --ignore-not-found + if [ -n "$local_port" ]; then + for alias in "${aliases[@]}"; do + attempt admin_discard /key/delete "$(ALIAS="$alias" yq --null-input --output-format json --indent 0 '{"key_aliases": [strenv(ALIAS)]}')" + done + attempt admin_discard /budget/delete "$(ID="$budget_id" yq --null-input --output-format json --indent 0 '{"id": strenv(ID)}')" + fi + [ -z "$forward" ] || kill "$forward" 2>/dev/null || true + attempt clear_secret_key + [ -z "$table_original" ] || attempt set_table "$table_original" + attempt flux resume kustomization litellm + end_deletions "$cell_status" +} +table_original="" budget_id="" +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +for deployment in litellm litellm-front; do + kubectl --namespace "$namespace" rollout status "deployment/$deployment" --timeout=5m +done + +table_original="$(kubectl --namespace "$namespace" get configmap litellm-keys -o jsonpath='{.data.keys\.yaml}')" +period="$(yq '.period' <<<"$table_original")" +person_budget="$(yq '.person_budget' <<<"$table_original")" +config_map="$(kubectl --namespace "$namespace" get deployment litellm \ + -o jsonpath='{.spec.template.spec.volumes[?(@.name=="config")].configMap.name}')" +budget_id="$(kubectl --namespace "$namespace" get configmap "$config_map" -o jsonpath='{.data.config\.yaml}' | + yq '.litellm_settings.max_end_user_budget_id')" +front_port="$(kubectl --namespace "$namespace" get service litellm-front -o jsonpath='{.spec.ports[0].port}')" +front_url="http://litellm-front.$namespace.svc.cluster.local:$front_port" + +master_key="$(kubectl --namespace "$namespace" get secret litellm -o jsonpath='{.data.master_key}' | base64 --decode)" +forbidden=() +litellm_port="$(kubectl --namespace "$namespace" get service litellm -o jsonpath='{.spec.ports[0].port}')" +forward_log="$RUNNER_TEMP/litellm-keys-port-forward.log" +kubectl --namespace "$namespace" port-forward service/litellm ":$litellm_port" >"$forward_log" 2>&1 & +forward=$! +for _ in $(seq 30); do grep -q '^Forwarding from 127.0.0.1:' "$forward_log" && break; sleep 1; done +local_port="$(sed -n 's/^Forwarding from 127\.0\.0\.1:\([0-9]*\) .*/\1/p' "$forward_log" | head -n 1)" +[ -n "$local_port" ] || fail "kubectl port-forward to the Service litellm did not start" +flux suspend kustomization litellm +# The empty copy of a fresh claim: LiteLLM migrates when it starts, so the budget list answers once it has. +for _ in $(seq 60); do admin /budget/list >/dev/null 2>&1 && break; sleep 2; done +[ "$(admin /budget/list | yq --input-format json 'length')" = 0 ] || fail "the cell expects a LiteLLM without budgets" + +# forbid : no output of a task may hold the value or its sha256. +forbid() { + local hash + hash="$(printf '%s' "$1" | sha256sum)" + forbidden+=("$1" "${hash%% *}") +} +forbid "$master_key" +# run_task ...: runs the task, which must end as expected; its output is +# checked for a value before anything of it is printed. +run_task() { + local case_name="$1" expected="$2" task_status=0 value + shift 2 + output="$(mise run "$@" 2>&1)" || task_status=$? + for value in "${forbidden[@]}"; do + case "$output" in *"$value"*) fail "$case_name: the output of $* holds a value, the master key or a value's sha256" ;; esac + done + [ "$task_status" = "$expected" ] || fail "$case_name: $* ended $task_status, not $expected: $output" +} +# snapshot: the app rows' fields and hashes, sorted, as one JSON line. +snapshot() { + admin '/key/list?return_full_object=true&size=100' | yq --input-format json --output-format json --indent 0 ' + [.keys[] | select(.key_alias == "hopin" or .key_alias == "presentator" or .key_alias == "songmaker") | + pick(["key_alias", "token", "max_budget", "budget_duration", "max_parallel_requests", "models", "allowed_routes", "budget_reset_at"])] | + sort_by(.key_alias)' +} +# row : a field of the app's row. +row() { + admin '/key/list?return_full_object=true&size=100' | + yq --input-format json ".keys[] | select(.key_alias == \"$1\") | .$2" +} + +# row_list : a list field of the app's row, sorted and joined. +row_list() { + admin '/key/list?return_full_object=true&size=100' | + yq --input-format json ".keys[] | select(.key_alias == \"$1\") | .$2 | sort | join(\",\")" +} +# The owners of the rows' models and routes: what LiteLLM lists, and the paths the front's rules match. +expected_models="$(admin /v1/models | yq --input-format json '[.data[].id] | sort | join(",")')" +front_routes_map="$(kubectl --namespace "$namespace" get deployment litellm-front \ + -o jsonpath='{.spec.template.spec.volumes[?(@.name=="routes")].configMap.name}')" +expected_routes="$(kubectl --namespace "$namespace" get configmap "$front_routes_map" -o jsonpath='{.data.route\.yml}' | + grep -o 'Path([`][^`]*[`])' | sed 's/^Path([`]//; s/[`])$//' | sort | paste -sd,)" +[ -n "$expected_models" ] && [ -n "$expected_routes" ] || fail "LiteLLM lists no model or the front's rules name no route" + +# equal_numbers : whether two numbers are equal as numbers, however each is written. +equal_numbers() { awk -v a="$1" -v b="$2" 'BEGIN { exit !(a != "" && b != "" && a + 0 == b + 0) }'; } + +# The table without hopin, whose key does not exist yet; rows made with the master key stand in for the loaded copy, +# with numbers other than the table's. +set_table "$(yq 'del(.apps.hopin)' <<<"$table_original")" +admin /budget/new "$(ID="$budget_id" yq --null-input --output-format json --indent 0 \ + '{"budget_id": strenv(ID), "max_budget": 99, "budget_duration": "1d"}')" >/dev/null +admin /key/generate '{"key_alias": "presentator", "max_budget": 7, "budget_duration": "1d", "max_parallel_requests": 2}' >/dev/null +admin /key/generate '{"key_alias": "songmaker", "max_budget": 3}' >/dev/null +before="$(snapshot)" + +run_task plan 2 litellm:keys --plan +grep -q "budget $budget_id max_budget 99 -> " <<<"$output" && grep -q 'key presentator max_budget 7 -> ' <<<"$output" && + grep -q 'key songmaker max_budget 3 -> ' <<<"$output" || fail "--plan did not list the differences: $output" +[ "$(snapshot)" = "$before" ] || fail "--plan changed a row" +equal_numbers "$(admin /budget/list | yq --input-format json ".[] | select(.budget_id == \"$budget_id\") | .max_budget")" 99 || + fail "--plan changed the person budget" +echo "--plan lists the differences, exits 2 and changes no row" + +run_task apply 0 litellm:keys +run_task second-apply 0 litellm:keys +grep -q ' 0 changes made' <<<"$output" || fail "a second run did not print 0 changes: $output" +run_task second-plan 0 litellm:keys --plan +for alias in presentator songmaker; do + equal_numbers "$(row "$alias" max_budget)" "$(ALIAS="$alias" yq '.apps[strenv(ALIAS)].budget' <<<"$table_original")" && + equal_numbers "$(row "$alias" max_parallel_requests)" "$(ALIAS="$alias" yq '.apps[strenv(ALIAS)].parallel' <<<"$table_original")" && + [ "$(row "$alias" budget_duration)" = "$period" ] || fail "$alias's row is not the table's" + [ "$(row_list "$alias" models)" = "$expected_models" ] && [ "$(row_list "$alias" allowed_routes)" = "$expected_routes" ] || + fail "$alias's row holds other models or routes than LiteLLM lists and the front's rules name" +done +budget_row="$(admin /budget/list | yq --input-format json ".[] | select(.budget_id == \"$budget_id\")")" +equal_numbers "$(yq '.max_budget' <<<"$budget_row")" "$person_budget" && [ "$(yq '.budget_duration' <<<"$budget_row")" = "$period" ] || + fail "the person budget $budget_id is not the table's $person_budget per $period" +echo "the table's numbers reach keys the command did not make and the person budget; a second run changes nothing" + +# A changed budget changes that field alone, and the period's reset stays. +reset_before="$(row presentator budget_reset_at)" +[[ $reset_before =~ ^[0-9]{4}- ]] || fail "presentator's row has no reset time: '$reset_before'" +raised="$(yq '.apps.presentator.budget + 1' <<<"$table_original")" +set_table "$(RAISED="$raised" yq 'del(.apps.hopin) | .apps.presentator.budget = (strenv(RAISED) | to_number)' <<<"$table_original")" +run_task budget-change 0 litellm:keys +[ "$(grep -c '^litellm:keys: key ' <<<"$output")" = 1 ] && grep -q "key presentator max_budget .* -> $raised\$" <<<"$output" || + fail "a changed budget did not change that field alone: $output" +equal_numbers "$(row presentator max_budget)" "$raised" && [ "$(row presentator budget_reset_at)" = "$reset_before" ] || + fail "the changed budget is not $raised, or the reset moved from $reset_before" +echo "a changed budget changes only that field and the reset stays" + +# A row without a key is named, the run ends 1 and the other rows are still made equal. +set_table "$table_original" +run_task row-without-key 1 litellm:keys +grep -q 'key hopin: none; run mise run litellm:key hopin' <<<"$output" || fail "the row without a key is not named: $output" +for alias in presentator songmaker; do + equal_numbers "$(row "$alias" max_budget)" "$(ALIAS="$alias" yq '.apps[strenv(ALIAS)].budget' <<<"$table_original")" || + fail "the run that named hopin's missing key did not make $alias's row equal to the table" +done +echo "a row without a key is named, the run ends 1 and the other rows are made equal" + +# litellm:key registers the value of the app's Secret; the Secret takes its key by stdin, so no argument holds it. +app_key="sk-$(openssl rand -hex 24)" +forbid "$app_key" +put_secret_key() { + printf '{"data":{"litellm_key":"%s"}}' "$(printf '%s' "$1" | base64 --wrap=0)" | + kubectl --namespace "$app_namespace" patch secret "$app_namespace" --type merge --patch-file /dev/stdin >/dev/null +} +put_secret_key "$app_key" +run_task registration 0 litellm:key hopin +equal_numbers "$(row hopin max_budget)" "$(yq '.apps.hopin.budget' <<<"$table_original")" || fail "hopin's row is not the table's" +run_task after-registration 0 litellm:keys --plan +grep -q ' 0 changes planned' <<<"$output" || fail "the new row differs from the table: $output" +client_pod="${CLIENT_POD//@NAME@/keys-client}" +client_pod="${client_pod//@IMAGE@/$CLIENT_IMAGE}" +# The header line reaches curl as a file of a Secret mounted in the Pod, so neither the Pod's arguments nor its stored +# specification hold the value, and the Secret takes it by stdin. +printf '{"apiVersion":"v1","kind":"Secret","metadata":{"name":"keys-client-header"},"data":{"header":"%s"}}' \ + "$(printf 'Authorization: Bearer %s\n' "$app_key" | base64 --wrap=0)" | + kubectl --namespace "$app_namespace" create -f - >/dev/null +client_pod="$(URL="$front_url/v1/models" yq ' + .spec.volumes = [{"name": "header", "secret": {"secretName": "keys-client-header", "defaultMode": 292}}] | + .spec.containers[0].volumeMounts = [{"name": "header", "mountPath": "/header", "readOnly": true}] | + .spec.containers[0].args = ["--silent", "--output", "/dev/null", "--write-out", "%{http_code}", + "--max-time", "10", "--retry", "5", "--retry-connrefused", "--retry-delay", "2", "--retry-max-time", "30", + "--header", "@/header/header", strenv(URL)]' <<<"$client_pod")" +printf '%s\n' "$client_pod" | kubectl --namespace "$app_namespace" create -f - >/dev/null +kubectl --namespace "$app_namespace" wait pod/keys-client --timeout=3m \ + --for=jsonpath='{.status.containerStatuses[0].state.terminated.exitCode}' >/dev/null +client_status="$(kubectl --namespace "$app_namespace" logs pod/keys-client)" +kubectl --namespace "$app_namespace" delete pod keys-client --wait --timeout=2m >/dev/null +[ "$client_status" = 200 ] || fail "the front answered a Pod in $app_namespace with $client_status, not 200, for the registered value" +echo "litellm:key makes the row with the table's fields and a Pod in $app_namespace gets 200 from the front with the value" + +run_task unchanged 0 litellm:key hopin +grep -q 'hopin unchanged' <<<"$output" || fail "a second litellm:key did not print unchanged: $output" +before="$(snapshot)" + +# Each refusal changes no row. +other_key="sk-$(openssl rand -hex 24)" +forbid "$other_key" +put_secret_key "$other_key" +run_task changed-value 1 litellm:key hopin +grep -q 'another key value holds the alias hopin' <<<"$output" || fail "a changed value was not refused as such: $output" +no_prefix_key="no-prefix-$(openssl rand -hex 24)" +forbid "$no_prefix_key" +put_secret_key "$no_prefix_key" +run_task no-prefix 1 litellm:key hopin +grep -q 'starting with sk-' <<<"$output" || fail "a value without sk- was not refused as such: $output" +clear_secret_key +run_task no-key 1 litellm:key hopin +grep -q 'has no litellm_key' <<<"$output" || fail "a Secret without the key was not refused as such: $output" +run_task outside-table 1 litellm:key not-in-the-table +grep -q 'has no row in the table' <<<"$output" || fail "an app outside the table was not refused as such: $output" +[ "$(snapshot)" = "$before" ] || fail "a refusal changed a row" +echo "a changed value, one without sk-, a Secret without the key and an app outside the table are refused and change no row" +echo "no output of any run holds a key's value, the master key or a value's sha256"