fix(core): arbitrate session startup

core/tests/gateway_oracle.rs

@@ -209,6 +209,39 @@ pub(crate) struct Oracle {
     pub(crate) ready: Ready,
 }
 
+struct OracleStartup {
+    child: Option<Child>,
+    directory: Option<std::path::PathBuf>,
+}
+
+impl OracleStartup {
+    fn new(directory: std::path::PathBuf) -> Self {
+        Self {
+            child: None,
+            directory: Some(directory),
+        }
+    }
+
+    fn finish(mut self, ready: Ready) -> Oracle {
+        Oracle {
+            child: self.child.take().expect("started oracle child"),
+            directory: self.directory.take().expect("oracle directory"),
+            ready,
+        }
+    }
+}
+
+impl Drop for OracleStartup {
+    fn drop(&mut self) {
+        if let Some(child) = self.child.as_mut() {
+            stop_child(child, self.directory.as_deref());
+        }
+        if let Some(directory) = self.directory.as_ref() {
+            let _ = fs::remove_dir_all(directory);
+        }
+    }
+}
+
 #[test]
 fn production_oracles_can_run_concurrently_with_isolated_state() {
     let (first_ready_tx, first_ready_rx) = std::sync::mpsc::channel();
@@ -235,10 +268,61 @@ fn production_oracles_can_run_concurrently_with_isolated_state() {
     assert!(concurrent, "second isolated oracle was globally serialized");
 }
 
+#[test]
+fn oracle_startup_timeout_kills_child_and_removes_directory() {
+    let directory = Oracle::unique_directory();
+    let mut command = Command::new("sh");
+    command.args(["-c", "while :; do sleep 1; done"]);
+    let started = Instant::now();
+
+    let result = Oracle::start_in("", directory.clone(), command, Duration::from_millis(20));
+
+    assert!(result.is_err(), "non-ready child unexpectedly became ready");
+    assert!(started.elapsed() < Duration::from_secs(2));
+    assert!(!directory.exists(), "timed-out oracle directory leaked");
+}
+
+#[test]
+fn oracle_source_failure_removes_directory_before_spawn() {
+    let directory = Oracle::unique_directory();
+    fs::create_dir(directory.join("main.go")).expect("block source file creation");
+
+    let result = Oracle::start_in(
+        "",
+        directory.clone(),
+        Command::new("go"),
+        Duration::from_millis(20),
+    );
+
+    assert!(result.is_err(), "invalid source path unexpectedly started");
+    assert!(!directory.exists(), "failed oracle directory leaked");
+}
+
+#[test]
+fn oracle_spawn_failure_removes_directory() {
+    let directory = Oracle::unique_directory();
+
+    let result = Oracle::start_in(
+        "",
+        directory.clone(),
+        Command::new("/definitely/missing/versevdi-go"),
+        Duration::from_millis(20),
+    );
+
+    assert!(result.is_err(), "missing executable unexpectedly started");
+    assert!(!directory.exists(), "spawn-failed oracle directory leaked");
+}
+
 impl Oracle {
     pub(crate) fn start(mode: &str) -> Self {
+        let directory = Self::unique_directory();
+        Self::start_in(mode, directory, Command::new("go"), Duration::from_secs(20))
+            .unwrap_or_else(|error| panic!("{error}"))
+    }
+
+    fn unique_directory() -> std::path::PathBuf {
         static NEXT: AtomicUsize = AtomicUsize::new(1);
-        let directory = loop {
+        loop {
             let candidate = std::env::temp_dir().join(format!(
                 "versevdi-rust-gateway-oracle-{}-{}",
                 std::process::id(),
@@ -249,13 +333,26 @@ impl Oracle {
                 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
                 Err(error) => panic!("create oracle directory: {error}"),
             }
-        };
+        }
+    }
+
+    fn start_in(
+        mode: &str,
+        directory: std::path::PathBuf,
+        mut command: Command,
+        timeout: Duration,
+    ) -> Result<Self, String> {
+        let mut startup = OracleStartup::new(directory.clone());
         let source = directory.join("main.go");
         let ready_path = directory.join("ready.json");
         let stop_path = directory.join("stop");
-        fs::write(&source, ORACLE_SOURCE).expect("write oracle source");
-        let child = Command::new("go")
-            .args(["run", source.to_str().expect("source path is UTF-8")])
+        fs::write(&source, ORACLE_SOURCE)
+            .map_err(|error| format!("write Go oracle source: {error}"))?;
+        let source = source
+            .to_str()
+            .ok_or_else(|| "Go oracle source path is not UTF-8".to_owned())?;
+        let child = command
+            .args(["run", source])
             .arg(&ready_path)
             .arg(&stop_path)
             .env("VERSEVDI_RUST_ORACLE_MODE", mode)
@@ -263,27 +360,29 @@ impl Oracle {
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
             .spawn()
-            .expect("start Go gateway oracle");
-        let deadline = Instant::now() + Duration::from_secs(20);
+            .map_err(|error| format!("start Go gateway oracle: {error}"))?;
+        startup.child = Some(child);
+        let deadline = Instant::now() + timeout;
         while Instant::now() < deadline {
             if let Ok(bytes) = fs::read(&ready_path) {
                 if let Ok(ready) = serde_json::from_slice(&bytes) {
-                    return Self {
-                        child,
-                        directory,
-                        ready,
-                    };
+                    return Ok(startup.finish(ready));
                 }
             }
+            if let Some(status) = startup
+                .child
+                .as_mut()
+                .expect("started oracle child")
+                .try_wait()
+                .map_err(|error| format!("poll Go gateway oracle: {error}"))?
+            {
+                return Err(format!(
+                    "Go gateway oracle exited before becoming ready: status={status}"
+                ));
+            }
             thread::sleep(Duration::from_millis(10));
         }
-        let output = child.wait_with_output().expect("collect Go oracle output");
-        panic!(
-            "Go gateway oracle did not become ready: status={} stdout={} stderr={}",
-            output.status,
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        );
+        Err("Go gateway oracle did not become ready before the startup deadline".to_owned())
     }
 
     fn provider_starts(&self) -> usize {
@@ -317,12 +416,34 @@ impl Oracle {
 
 impl Drop for Oracle {
     fn drop(&mut self) {
-        let _ = fs::write(self.directory.join("stop"), []);
-        let _ = self.child.wait();
+        stop_child(&mut self.child, Some(&self.directory));
         let _ = fs::remove_dir_all(&self.directory);
     }
 }
 
+fn stop_child(child: &mut Child, directory: Option<&std::path::Path>) {
+    if let Some(directory) = directory {
+        let _ = fs::write(directory.join("stop"), []);
+    }
+    if wait_for_child(child, Duration::from_millis(250)) {
+        return;
+    }
+    if child.kill().is_ok() {
+        let _ = wait_for_child(child, Duration::from_millis(500));
+    }
+}
+
+fn wait_for_child(child: &mut Child, timeout: Duration) -> bool {
+    let deadline = Instant::now() + timeout;
+    loop {
+        match child.try_wait() {
+            Ok(Some(_)) => return true,
+            Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(5)),
+            Ok(None) | Err(_) => return false,
+        }
+    }
+}
+
 pub(crate) fn test_key(encoded: &str) -> Arc<dyn SigningKey> {
     let der = STANDARD.decode(encoded).expect("decode test key");
     rustls::crypto::ring::default_provider()