diff --git a/docker/docker_jit_monitor/src/github_api.rs b/docker/docker_jit_monitor/src/github_api.rs index 861650c..24b2d9a 100644 --- a/docker/docker_jit_monitor/src/github_api.rs +++ b/docker/docker_jit_monitor/src/github_api.rs @@ -1,4 +1,4 @@ -use std::process::{Command, Output}; +use std::process::{Command, ExitCode, Output}; use log::{debug, warn}; use serde::Deserialize; @@ -97,6 +97,13 @@ pub(crate) fn spawn_runner(config: RunnerConfig) -> Result, +} + +impl Retries { + /// Increases the number of retries and quits if it reaches `MAX_SPAWN_RETRIES`. + fn inc_and_check(&mut self, t: ContainerType) { + let value = self.t.entry(t).or_insert(0); + *value += 1; + if *value > MAX_SPAWN_RETRIES { + println!( + "We had {value} many times to spawn a runner/builder ({t:?}). It is not happening." + ); + std::process::exit(-1); + } + } + + /// Resets the counter when we have succesfully spawned a runner. + fn reset(&mut self, t: ContainerType) { + self.t.entry(t).insert_entry(0); + } + + /// The current wait time we have for a loop. + /// Defaults to `BASE_LOOP_SLEEP` and exponentially increases with failures. + fn wait_time(&self) -> Duration { + let m = self.t.values().max().unwrap_or(&0); + Duration::from_millis(BASE_LOOP_SLEEP * 2_u64.pow(*m)) + } +} + fn main() -> anyhow::Result<()> { env_logger::init(); info!("Starting monitor for selfhosted docker-based github runners!"); @@ -232,6 +271,7 @@ fn main() -> anyhow::Result<()> { let mut running_containers: Vec = vec![]; // Todo: implement something to reserve devices for the duration of the docker run child process. + let mut retries = Retries::default(); loop { let exiting = EXITING.load(Ordering::Relaxed); for container_type in ContainerType::iter() { @@ -254,13 +294,17 @@ fn main() -> anyhow::Result<()> { }; match spawn_runner(config) { - Ok(container) => running_containers.push(container), + Ok(container) => { + retries.reset(container_type); + running_containers.push(container) + } Err(SpawnRunnerError::GhApiError(_, message)) if message.contains("gh: Already exists") => { info!("Runner name already taken - Will retry with new name later") } Err(e) => { + retries.inc_and_check(container_type); error!("Failed to spawn JIT runner: {e:?}"); } } @@ -319,7 +363,7 @@ fn main() -> anyhow::Result<()> { } running_containers = still_running; - thread::sleep(Duration::from_millis(500)); + thread::sleep(retries.wait_time()); } info!("Exiting....");